You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/06/08 05:03:16 UTC

[JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.8.0_05) - Build # 10502 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10502/
Java: 32bit/jdk1.8.0_05 -server -XX:+UseParallelGC

1 tests failed.
FAILED:  org.apache.solr.cloud.ShardSplitTest.testDistribSearch

Error Message:
collection already exists: routeFieldColl

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: collection already exists: routeFieldColl
	at __randomizedtesting.SeedInfo.seed([DA8AE7D41FA8DFD6:5B6C69CC68F7BFEA]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:554)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.doRequest(LBHttpSolrServer.java:340)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:301)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:650)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1505)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1460)
	at org.apache.solr.cloud.ShardSplitTest.splitByRouteFieldTest(ShardSplitTest.java:263)
	at org.apache.solr.cloud.ShardSplitTest.doTest(ShardSplitTest.java:105)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
	at sun.reflect.GeneratedMethodAccessor49.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11734 lines...]
   [junit4] Suite: org.apache.solr.cloud.ShardSplitTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/init-core-data-001
   [junit4]   2> 1290135 T6349 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /x/jy
   [junit4]   2> 1290138 T6349 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 1290139 T6349 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1290139 T6350 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1290239 T6349 oasc.ZkTestServer.run start zk server on port:43043
   [junit4]   2> 1290241 T6349 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1291665 T6353 oazsp.FileTxnLog.commit WARN fsync-ing the write ahead log in SyncThread:0 took 1423ms which will adversely effect operation latency. See the ZooKeeper troubleshooting guide
   [junit4]   2> 1291667 T6356 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@843a96 name:ZooKeeperConnection Watcher:127.0.0.1:43043 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1291667 T6349 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1291668 T6349 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1291674 T6349 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1291675 T6358 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@61b4d0 name:ZooKeeperConnection Watcher:127.0.0.1:43043/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1291676 T6349 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1291676 T6349 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1291679 T6349 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1291683 T6349 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1291685 T6349 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1291688 T6349 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1291690 T6349 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1291694 T6349 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1291695 T6349 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1291698 T6349 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1291699 T6349 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1291705 T6349 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1291706 T6349 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1291710 T6349 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1291711 T6349 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1291714 T6349 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1291715 T6349 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1291718 T6349 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1291719 T6349 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1291721 T6349 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1291722 T6349 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1291725 T6349 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1291726 T6349 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1291729 T6349 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1291730 T6349 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1291733 T6349 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1291733 T6349 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1291984 T6349 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1291989 T6349 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:48408
   [junit4]   2> 1291990 T6349 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1291990 T6349 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1291991 T6349 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-002
   [junit4]   2> 1291991 T6349 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-002/'
   [junit4]   2> 1292002 T6349 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-002/solr.xml
   [junit4]   2> 1292010 T6349 oasc.CoreContainer.<init> New CoreContainer 9566476
   [junit4]   2> 1292010 T6349 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-002/]
   [junit4]   2> 1292011 T6349 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1292011 T6349 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1292011 T6349 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1292012 T6349 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1292012 T6349 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1292013 T6349 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1292013 T6349 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1292013 T6349 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1292013 T6349 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1292014 T6349 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1292014 T6349 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1292015 T6349 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1292015 T6349 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:43043/solr
   [junit4]   2> 1292015 T6349 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1292017 T6349 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1292018 T6369 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5486f5 name:ZooKeeperConnection Watcher:127.0.0.1:43043 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1292018 T6349 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1292019 T6349 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1292020 T6371 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5cc9f9 name:ZooKeeperConnection Watcher:127.0.0.1:43043/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1292020 T6349 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1292021 T6349 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1292023 T6349 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1292024 T6349 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1292025 T6349 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1292027 T6349 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1292029 T6349 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1292030 T6349 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48408_x%2Fjy
   [junit4]   2> 1292031 T6349 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48408_x%2Fjy
   [junit4]   2> 1292035 T6349 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1292037 T6349 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1292040 T6349 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:48408_x%2Fjy
   [junit4]   2> 1292040 T6349 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1292042 T6349 oasc.Overseer.start Overseer (id=91894339924393987-127.0.0.1:48408_x%2Fjy-n_0000000000) starting
   [junit4]   2> 1292044 T6349 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1292051 T6373 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1292052 T6349 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1292055 T6349 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1292057 T6349 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1292060 T6372 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1292062 T6374 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1292063 T6374 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1292064 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1292064 T6374 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1292066 T6372 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1292067 T6372 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48408/x/jy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:48408_x%2Fjy",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1292067 T6372 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 1292067 T6372 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1292070 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1292072 T6371 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1293065 T6374 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1293065 T6374 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1293066 T6374 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1293067 T6374 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1293068 T6374 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1293068 T6374 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-002/collection1/'
   [junit4]   2> 1293069 T6374 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 1293070 T6374 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 1293070 T6374 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 1293097 T6374 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 1293105 T6374 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1293106 T6374 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1293110 T6374 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1293184 T6374 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1293185 T6374 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1293186 T6374 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1293188 T6374 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1293204 T6374 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1293204 T6374 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1293204 T6374 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-001/control/data/
   [junit4]   2> 1293205 T6374 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@d852b
   [junit4]   2> 1293206 T6374 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-001/control/data
   [junit4]   2> 1293207 T6374 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-001/control/data/index/
   [junit4]   2> 1293207 T6374 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1293208 T6374 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-001/control/data/index
   [junit4]   2> 1293208 T6374 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=18, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1293209 T6374 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@16cddc7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@92c524),segFN=segments_1,generation=1}
   [junit4]   2> 1293209 T6374 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1293213 T6374 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1293213 T6374 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1293214 T6374 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1293214 T6374 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1293214 T6374 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1293215 T6374 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1293215 T6374 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1293216 T6374 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1293216 T6374 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1293217 T6374 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1293217 T6374 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1293218 T6374 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1293218 T6374 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1293219 T6374 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1293219 T6374 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1293220 T6374 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1293220 T6374 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1293223 T6374 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1293225 T6374 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1293226 T6374 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1293227 T6374 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=13.015625, floorSegmentMB=0.7294921875, forceMergeDeletesPctAllowed=22.950742376693494, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1293227 T6374 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@16cddc7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@92c524),segFN=segments_1,generation=1}
   [junit4]   2> 1293228 T6374 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1293228 T6374 oass.SolrIndexSearcher.<init> Opening Searcher@eb924b[collection1] main
   [junit4]   2> 1293229 T6374 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1293230 T6374 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1293230 T6374 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1293230 T6374 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1293231 T6374 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1293231 T6374 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1293231 T6374 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1293232 T6374 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1293232 T6374 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1293232 T6374 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1293233 T6375 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@eb924b[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 1293234 T6374 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1293235 T6378 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:48408/x/jy collection:control_collection shard:shard1
   [junit4]   2> 1293235 T6349 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1293235 T6378 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1293235 T6349 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1293238 T6349 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1293239 T6380 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14baf6c name:ZooKeeperConnection Watcher:127.0.0.1:43043/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1293240 T6349 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1293241 T6349 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1293243 T6378 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1293244 T6349 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1293244 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1293245 T6378 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1293245 T6378 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1847 name=collection1 org.apache.solr.core.SolrCore@1707b9b url=http://127.0.0.1:48408/x/jy/collection1 node=127.0.0.1:48408_x%2Fjy C1847_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:48408/x/jy, core=collection1, node_name=127.0.0.1:48408_x%2Fjy}
   [junit4]   2> 1293245 T6378 C1847 P48408 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:48408/x/jy/collection1/
   [junit4]   2> 1293246 T6372 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1293246 T6378 C1847 P48408 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1293247 T6378 C1847 P48408 oasc.SyncStrategy.syncToMe http://127.0.0.1:48408/x/jy/collection1/ has no replicas
   [junit4]   2> 1293247 T6378 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:48408/x/jy/collection1/ shard1
   [junit4]   2> 1293247 T6378 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1293248 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1293252 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1293255 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1293357 T6371 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1293357 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1293403 T6378 oasc.ZkController.register We are http://127.0.0.1:48408/x/jy/collection1/ and leader is http://127.0.0.1:48408/x/jy/collection1/
   [junit4]   2> 1293403 T6378 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:48408/x/jy
   [junit4]   2> 1293403 T6378 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1293404 T6378 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1293404 T6378 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1293406 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1293406 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1293406 T6378 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1293406 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1293407 T6372 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1293407 T6372 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48408/x/jy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:48408_x%2Fjy",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1293409 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1293479 T6349 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1293479 T6349 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1293481 T6349 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:59932
   [junit4]   2> 1293482 T6349 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1293482 T6349 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1293482 T6349 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-003
   [junit4]   2> 1293483 T6349 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-003/'
   [junit4]   2> 1293496 T6349 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-003/solr.xml
   [junit4]   2> 1293503 T6349 oasc.CoreContainer.<init> New CoreContainer 33207913
   [junit4]   2> 1293503 T6349 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-003/]
   [junit4]   2> 1293504 T6349 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1293504 T6349 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1293504 T6349 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1293504 T6349 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1293505 T6349 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1293505 T6349 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1293505 T6349 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1293506 T6349 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1293506 T6349 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1293507 T6349 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1293507 T6349 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1293507 T6349 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1293508 T6349 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:43043/solr
   [junit4]   2> 1293508 T6349 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1293509 T6349 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1293514 T6391 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@423fd3 name:ZooKeeperConnection Watcher:127.0.0.1:43043 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1293514 T6349 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1293514 T6371 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1293514 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1293516 T6349 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1293517 T6393 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e1badf name:ZooKeeperConnection Watcher:127.0.0.1:43043/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1293517 T6349 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1293521 T6349 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1294523 T6349 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59932_x%2Fjy
   [junit4]   2> 1294524 T6349 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:59932_x%2Fjy
   [junit4]   2> 1294530 T6394 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1294530 T6394 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1294531 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1294531 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1294531 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1294531 T6394 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1294532 T6372 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1294532 T6372 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59932/x/jy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:59932_x%2Fjy",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1294532 T6372 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 1294533 T6372 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1294534 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1294636 T6393 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1294636 T6371 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1294636 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1295532 T6394 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1295532 T6394 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1295532 T6394 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1295533 T6394 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1295533 T6394 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1295533 T6394 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-003/collection1/'
   [junit4]   2> 1295534 T6394 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 1295534 T6394 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 1295534 T6394 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 1295551 T6394 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 1295561 T6394 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1295562 T6394 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1295568 T6394 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1295645 T6394 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1295647 T6394 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1295648 T6394 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1295649 T6394 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1295659 T6394 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1295659 T6394 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1295659 T6394 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-001/jetty1/
   [junit4]   2> 1295659 T6394 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@d852b
   [junit4]   2> 1295660 T6394 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-001/jetty1
   [junit4]   2> 1295661 T6394 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-001/jetty1/index/
   [junit4]   2> 1295661 T6394 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1295661 T6394 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-001/jetty1/index
   [junit4]   2> 1295661 T6394 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=18, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1295662 T6394 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@127d897 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@31fe46),segFN=segments_1,generation=1}
   [junit4]   2> 1295662 T6394 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1295664 T6394 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1295664 T6394 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1295665 T6394 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1295665 T6394 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1295665 T6394 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1295665 T6394 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1295665 T6394 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1295666 T6394 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1295666 T6394 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1295666 T6394 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1295666 T6394 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1295667 T6394 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1295667 T6394 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1295667 T6394 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1295667 T6394 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1295668 T6394 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1295668 T6394 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1295669 T6394 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1295670 T6394 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1295671 T6394 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1295671 T6394 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=13.015625, floorSegmentMB=0.7294921875, forceMergeDeletesPctAllowed=22.950742376693494, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1295672 T6394 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@127d897 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@31fe46),segFN=segments_1,generation=1}
   [junit4]   2> 1295672 T6394 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1295672 T6394 oass.SolrIndexSearcher.<init> Opening Searcher@12a80c9[collection1] main
   [junit4]   2> 1295672 T6394 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1295673 T6394 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1295673 T6394 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1295674 T6394 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1295674 T6394 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1295674 T6394 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1295674 T6394 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1295675 T6394 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1295675 T6394 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1295675 T6394 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1295677 T6395 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@12a80c9[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 1295678 T6394 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1295679 T6398 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:59932/x/jy collection:collection1 shard:shard2
   [junit4]   2> 1295679 T6349 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1295680 T6349 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1295679 T6398 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1295724 T6398 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1295925 T6349 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 1295926 T6349 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1295930 T6349 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:59619
   [junit4]   2> 1295931 T6349 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1295932 T6349 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1295933 T6349 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-004
   [junit4]   2> 1295933 T6349 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-004/'
   [junit4]   2> 1295952 T6349 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-004/solr.xml
   [junit4]   2> 1295959 T6349 oasc.CoreContainer.<init> New CoreContainer 24608912
   [junit4]   2> 1295960 T6349 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-004/]
   [junit4]   2> 1295960 T6349 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1295960 T6349 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1295961 T6349 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1295961 T6349 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1295961 T6349 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1295962 T6349 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1295962 T6349 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1295962 T6349 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1295963 T6349 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1295963 T6349 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1295963 T6349 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1295964 T6349 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1295964 T6349 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:43043/solr
   [junit4]   2> 1295965 T6349 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1295965 T6349 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1299538 T6353 oazsp.FileTxnLog.commit WARN fsync-ing the write ahead log in SyncThread:0 took 3793ms which will adversely effect operation latency. See the ZooKeeper troubleshooting guide
   [junit4]   2> 1299539 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1299539 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1299539 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1299539 T6409 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a2de38 name:ZooKeeperConnection Watcher:127.0.0.1:43043 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1299540 T6372 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1299540 T6398 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1299540 T6398 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1848 name=collection1 org.apache.solr.core.SolrCore@71c304 url=http://127.0.0.1:59932/x/jy/collection1 node=127.0.0.1:59932_x%2Fjy C1848_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:59932/x/jy, core=collection1, node_name=127.0.0.1:59932_x%2Fjy}
   [junit4]   2> 1299541 T6398 C1848 P59932 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:59932/x/jy/collection1/
   [junit4]   2> 1299541 T6398 C1848 P59932 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1299540 T6349 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1299542 T6393 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1299541 T6398 C1848 P59932 oasc.SyncStrategy.syncToMe http://127.0.0.1:59932/x/jy/collection1/ has no replicas
   [junit4]   2> 1299542 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1299542 T6398 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:59932/x/jy/collection1/ shard2
   [junit4]   2> 1299542 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1299543 T6398 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1299543 T6371 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1299544 T6349 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1299544 T6411 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c2054 name:ZooKeeperConnection Watcher:127.0.0.1:43043/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1299545 T6349 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1299547 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1299547 T6372 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1299549 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1299550 T6349 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1299652 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1299652 T6371 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1299652 T6393 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1299652 T6411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1299697 T6398 oasc.ZkController.register We are http://127.0.0.1:59932/x/jy/collection1/ and leader is http://127.0.0.1:59932/x/jy/collection1/
   [junit4]   2> 1299698 T6398 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:59932/x/jy
   [junit4]   2> 1299698 T6398 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1299698 T6398 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1299698 T6398 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1299700 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1299700 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1299700 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1299701 T6398 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1299701 T6372 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1299701 T6372 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59932/x/jy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:59932_x%2Fjy",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1299703 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1299804 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1299804 T6371 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1299804 T6411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1299804 T6393 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1300552 T6349 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59619_x%2Fjy
   [junit4]   2> 1300554 T6349 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:59619_x%2Fjy
   [junit4]   2> 1300559 T6412 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1300560 T6412 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1300560 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1300560 T6412 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1300560 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1300561 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1300561 T6372 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1300561 T6372 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59619/x/jy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:59619_x%2Fjy",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1300561 T6372 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 1300562 T6372 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1300563 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1300664 T6411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1300664 T6393 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1300664 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1300664 T6371 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1301561 T6412 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1301561 T6412 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1301561 T6412 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1301561 T6412 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1301562 T6412 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1301562 T6412 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-004/collection1/'
   [junit4]   2> 1301563 T6412 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 1301563 T6412 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 1301564 T6412 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 1301579 T6412 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 1301593 T6412 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1301594 T6412 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1301598 T6412 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1301669 T6412 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1301671 T6412 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1301671 T6412 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1301673 T6412 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1301683 T6412 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1301684 T6412 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1301684 T6412 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-001/jetty2/
   [junit4]   2> 1301684 T6412 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@d852b
   [junit4]   2> 1301685 T6412 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-001/jetty2
   [junit4]   2> 1301685 T6412 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-001/jetty2/index/
   [junit4]   2> 1301685 T6412 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1301686 T6412 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-001/jetty2/index
   [junit4]   2> 1301686 T6412 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=18, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1301686 T6412 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1af6f43 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@d475ad),segFN=segments_1,generation=1}
   [junit4]   2> 1301687 T6412 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1301689 T6412 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1301689 T6412 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1301689 T6412 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1301689 T6412 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1301690 T6412 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1301690 T6412 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1301690 T6412 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1301690 T6412 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1301690 T6412 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1301691 T6412 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1301691 T6412 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1301691 T6412 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1301691 T6412 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1301692 T6412 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1301692 T6412 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1301692 T6412 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1301693 T6412 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1301694 T6412 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1301695 T6412 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1301695 T6412 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1301696 T6412 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=13.015625, floorSegmentMB=0.7294921875, forceMergeDeletesPctAllowed=22.950742376693494, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1301696 T6412 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1af6f43 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@d475ad),segFN=segments_1,generation=1}
   [junit4]   2> 1301696 T6412 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1301697 T6412 oass.SolrIndexSearcher.<init> Opening Searcher@136637[collection1] main
   [junit4]   2> 1301697 T6412 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1301697 T6412 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1301698 T6412 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1301698 T6412 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1301698 T6412 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1301698 T6412 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1301699 T6412 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1301699 T6412 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1301699 T6412 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1301699 T6412 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1301701 T6413 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@136637[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 1301702 T6412 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1301703 T6416 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:59619/x/jy collection:collection1 shard:shard1
   [junit4]   2> 1301703 T6349 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1301704 T6349 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1301703 T6416 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1301709 T6416 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1301710 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1301710 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1301710 T6416 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1301710 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1301710 T6416 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1301711 T6372 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C1849 name=collection1 org.apache.solr.core.SolrCore@114559a url=http://127.0.0.1:59619/x/jy/collection1 node=127.0.0.1:59619_x%2Fjy C1849_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:59619/x/jy, core=collection1, node_name=127.0.0.1:59619_x%2Fjy}
   [junit4]   2> 1301711 T6416 C1849 P59619 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:59619/x/jy/collection1/
   [junit4]   2> 1301711 T6416 C1849 P59619 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1301711 T6416 C1849 P59619 oasc.SyncStrategy.syncToMe http://127.0.0.1:59619/x/jy/collection1/ has no replicas
   [junit4]   2> 1301712 T6416 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:59619/x/jy/collection1/ shard1
   [junit4]   2> 1301712 T6416 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1301712 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1301714 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1301715 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1301816 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1301816 T6411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1301816 T6393 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1301816 T6371 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1301865 T6416 oasc.ZkController.register We are http://127.0.0.1:59619/x/jy/collection1/ and leader is http://127.0.0.1:59619/x/jy/collection1/
   [junit4]   2> 1301865 T6416 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:59619/x/jy
   [junit4]   2> 1301865 T6416 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1301865 T6416 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1301866 T6416 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1301867 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1301867 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1301867 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1301867 T6416 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1301868 T6372 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1301868 T6372 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59619/x/jy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:59619_x%2Fjy",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 1301870 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1301945 T6349 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 1301946 T6349 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1301955 T6349 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54088
   [junit4]   2> 1301956 T6349 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1301956 T6349 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1301957 T6349 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-005
   [junit4]   2> 1301957 T6349 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-005/'
   [junit4]   2> 1301967 T6349 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-005/solr.xml
   [junit4]   2> 1301972 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1301972 T6393 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1301972 T6371 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1301972 T6411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1301975 T6349 oasc.CoreContainer.<init> New CoreContainer 30426829
   [junit4]   2> 1301975 T6349 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001/tempDir-005/]
   [junit4]   2> 1301976 T6349 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1301976 T6349 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1301976 T6349 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1301977 T6349 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1301977 T6349 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1301977 T6349 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1301978 T6349 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1301978 T6349 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1301978 T6349 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1301979 T6349 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1301979 T6349 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1301979 T6349 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1301980 T6349 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:43043/solr
   [junit4]   2> 1301980 T6349 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1301981 T6349 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1301982 T6427 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d2e03 name:ZooKeeperConnection Watcher:127.0.0.1:43043 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1301982 T6349 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1301983 T6349 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1301984 T6429 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1893615 name:ZooKeeperConnection Watcher:127.0.0.1:43043/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1301984 T6349 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1301988 T6349 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1302990 T6349 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54088_x%2Fjy
   [junit4]   2> 1302992 T6349 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54088_x%2Fjy
   [junit4]   2> 1303002 T6430 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1303002 T6430 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1303004 T6371 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1303004 T6430 oasc.ZkController.waitForCoreNodeName look for our core node name
   [ju

[...truncated too long message...]

pache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 1493034 T6449 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 1493034 T6449 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/routeFieldColl/leader_elect/shard2/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ShardSplitTest-DA8AE7D41FA8DFD6-001
   [junit4]   2> NOTE: test params are: codec=Lucene46, sim=DefaultSimilarity, locale=is, timezone=Pacific/Fiji
   [junit4]   2> NOTE: Linux 3.8.0-41-generic i386/Oracle Corporation 1.8.0_05 (32-bit)/cpus=8,threads=1,free=304172784,total=518258688
   [junit4]   2> NOTE: All tests run in this JVM: [TestWordDelimiterFilterFactory, MigrateRouteKeyTest, TestCloudManagedSchema, DirectSolrConnectionTest, AlternateDirectoryTest, HdfsDirectoryTest, MultiTermTest, TestReload, TestLMDirichletSimilarityFactory, TestInfoStreamLogging, TestRangeQuery, TestReplicationHandlerBackup, OpenExchangeRatesOrgProviderTest, OpenCloseCoreStressTest, SuggestComponentTest, TestSolrXMLSerializer, TestManagedStopFilterFactory, DocValuesMultiTest, HighlighterConfigTest, XsltUpdateRequestHandlerTest, DeleteInactiveReplicaTest, ConvertedLegacyTest, TestSolr4Spatial, StandardRequestHandlerTest, TestOmitPositions, TestDocBasedVersionConstraints, TestChildDocTransformer, TestDFRSimilarityFactory, DisMaxRequestHandlerTest, TestNoOpRegenerator, TestShardHandlerFactory, BasicZkTest, SuggesterWFSTTest, TestBinaryResponseWriter, TestFastWriter, NumericFieldsTest, LegacyHTMLStripCharFilterTest, OverseerRolesTest, BlockDirectoryTest, TestDistributedSearch, RemoteQueryErrorTest, PreAnalyzedUpdateProcessorTest, TestDynamicFieldCollectionResource, SSLMigrationTest, TestHashPartitioner, TestUpdate, SuggesterTSTTest, TestInitQParser, TestFuzzyAnalyzedSuggestions, SignatureUpdateProcessorFactoryTest, SortByFunctionTest, TestCloudInspectUtil, TestStressLucene, TestExpandComponent, TestManagedSynonymFilterFactory, HighlighterTest, NoCacheHeaderTest, TestRequestStatusCollectionAPI, FunctionTest, TestCSVLoader, TestLRUCache, TestSolrXmlPersistence, TestBinaryField, DirectSolrSpellCheckerTest, TestSolrDeletionPolicy2, TestFoldingMultitermQuery, SolrIndexConfigTest, AbstractAnalyticsStatsTest, AnalysisErrorHandlingTest, HdfsRecoveryZkTest, ResourceLoaderTest, TestRTGBase, SolrPluginUtilsTest, TestDefaultSearchFieldResource, BadCopyFieldTest, TestDynamicFieldResource, CollectionsAPIDistributedZkTest, TestCoreDiscovery, PluginInfoTest, StatsComponentTest, TestFieldResource, SolrXmlInZkTest, JSONWriterTest, TestSolrCoreProperties, FullSolrCloudDistribCmdsTest, DocumentAnalysisRequestHandlerTest, TestHighlightDedupGrouping, TestRecoveryHdfs, CoreMergeIndexesAdminHandlerTest, AtomicUpdatesTest, RequiredFieldsTest, XmlUpdateRequestHandlerTest, TestQuerySenderListener, TestZkChroot, TestClassNameShortening, TestQueryTypes, HardAutoCommitTest, DistribDocExpirationUpdateProcessorTest, TestIndexSearcher, ZkSolrClientTest, RankQueryTest, DistanceFunctionTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, LeaderElectionIntegrationTest, LeaderElectionTest, ZkCLITest, ShardRoutingCustomTest, SliceStateUpdateTest, AutoCommitTest, TestDistributedGrouping, TestFaceting, TestRecovery, TermVectorComponentDistributedTest, TestStressReorder, SpellCheckComponentTest, SolrCmdDistributorTest, QueryElevationComponentTest, BadIndexSchemaTest, TestFiltering, CoreAdminHandlerTest, DocValuesTest, WordBreakSolrSpellCheckerTest, TestPseudoReturnFields, FieldMutatingUpdateProcessorTest, TestAtomicUpdateErrorCases, TestSolrDeletionPolicy1, DebugComponentTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, TestWriterPerf, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, RegexBoostProcessorTest, TestJmxIntegration, UpdateRequestProcessorFactoryTest, MBeansHandlerTest, TestAnalyzedSuggestions, JsonLoaderTest, PingRequestHandlerTest, TestComponentsName, CoreContainerCoreInitFailuresTest, UpdateParamsTest, TestQuerySenderNoQuery, BadComponentTest, SampleTest, MinimalSchemaTest, TestElisionMultitermQuery, TestConfig, OutputWriterTest, NotRequiredUniqueKeyTest, TestPhraseSuggestions, TestCharFilters, TestCodecSupport, SynonymTokenizerTest, TestXIncludeConfig, EchoParamsTest, TestSweetSpotSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestIBSimilarityFactory, TestBM25SimilarityFactory, TestDefaultSimilarityFactory, TestJmxMonitoredMap, TestFastOutputStream, URLClassifyProcessorTest, DateMathParserTest, TestSystemIdResolver, PrimUtilsTest, DateFieldTest, SpellingQueryConverterTest, DOMUtilTest, TestSolrJ, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, SystemInfoHandlerTest, FileUtilsTest, CursorPagingTest, ExpressionTest, FieldFacetExtrasTest, CollectionsAPIAsyncDistributedZkTest, ConnectionManagerTest, DeleteReplicaTest, OverseerStatusTest, ShardSplitTest]
   [junit4] Completed on J1 in 203.71s, 1 test, 1 error <<< FAILURES!

[...truncated 133 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 410 suites, 1693 tests, 1 error, 36 ignored (15 assumptions)

Total time: 61 minutes 55 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.8.0_05 -server -XX:+UseParallelGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



Re: [JENKINS] Lucene-Solr-trunk-Linux (64bit/jdk1.8.0_20-ea-b15) - Build # 10516 - Still Failing!

Posted by "david.w.smiley@gmail.com" <da...@gmail.com>.
I’m on it.

~ David Smiley
Freelance Apache Lucene/Solr Search Consultant/Developer
http://www.linkedin.com/in/davidwsmiley


On Mon, Jun 9, 2014 at 10:36 AM, Policeman Jenkins Server <
jenkins@thetaphi.de> wrote:

> Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10516/
> Java: 64bit/jdk1.8.0_20-ea-b15 -XX:-UseCompressedOops -XX:+UseParallelGC
>
> 1 tests failed.
> FAILED:  org.apache.lucene.spatial.prefix.DateNRStrategyTest.testWithin
> {#4 seed=[3F4202D795A1E146:B3DDDC492CA0D99]}
>
> Error Message:
> Shouldn't match I#2:[-264000 TO -264000-11-20] Q:-264000
>
> Stack Trace:
> java.lang.AssertionError: Shouldn't match I#2:[-264000 TO -264000-11-20]
> Q:-264000
>         at
> __randomizedtesting.SeedInfo.seed([3F4202D795A1E146:B3DDDC492CA0D99]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at
> org.apache.lucene.spatial.prefix.BaseNonFuzzySpatialOpStrategyTest.fail(BaseNonFuzzySpatialOpStrategyTest.java:128)
>         at
> org.apache.lucene.spatial.prefix.BaseNonFuzzySpatialOpStrategyTest.testOperation(BaseNonFuzzySpatialOpStrategyTest.java:117)
>         at
> org.apache.lucene.spatial.prefix.BaseNonFuzzySpatialOpStrategyTest.testOperationRandomShapes(BaseNonFuzzySpatialOpStrategyTest.java:64)
>         at
> org.apache.lucene.spatial.prefix.DateNRStrategyTest.testWithin(DateNRStrategyTest.java:59)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>         at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:483)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at
> org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at
> org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at
> org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>         at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at
> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
>         at java.lang.Thread.run(Thread.java:745)
>
>
>
>
> Build Log:
> [...truncated 9391 lines...]
>    [junit4] Suite: org.apache.lucene.spatial.prefix.DateNRStrategyTest
>    [junit4]   2> NOTE: reproduce with: ant test
>  -Dtestcase=DateNRStrategyTest -Dtests.method=testWithin
> -Dtests.seed=3F4202D795A1E146 -Dtests.multiplier=3 -Dtests.slow=true
> -Dtests.locale=ga_IE -Dtests.timezone=America/Argentina/Jujuy
> -Dtests.file.encoding=ISO-8859-1
>    [junit4] FAILURE 0.28s J1 | DateNRStrategyTest.testWithin {#4
> seed=[3F4202D795A1E146:B3DDDC492CA0D99]} <<<
>    [junit4]    > Throwable #1: java.lang.AssertionError: Shouldn't match
> I#2:[-264000 TO -264000-11-20] Q:-264000
>    [junit4]    >        at
> __randomizedtesting.SeedInfo.seed([3F4202D795A1E146:B3DDDC492CA0D99]:0)
>    [junit4]    >        at
> org.apache.lucene.spatial.prefix.BaseNonFuzzySpatialOpStrategyTest.fail(BaseNonFuzzySpatialOpStrategyTest.java:128)
>    [junit4]    >        at
> org.apache.lucene.spatial.prefix.BaseNonFuzzySpatialOpStrategyTest.testOperation(BaseNonFuzzySpatialOpStrategyTest.java:117)
>    [junit4]    >        at
> org.apache.lucene.spatial.prefix.BaseNonFuzzySpatialOpStrategyTest.testOperationRandomShapes(BaseNonFuzzySpatialOpStrategyTest.java:64)
>    [junit4]    >        at
> org.apache.lucene.spatial.prefix.DateNRStrategyTest.testWithin(DateNRStrategyTest.java:59)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2> NOTE: leaving temporary files on disk at:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/build/spatial/test/J1/./temp/lucene.spatial.prefix.DateNRStrategyTest-3F4202D795A1E146-001
>    [junit4]   2> NOTE: test params are: codec=Lucene40,
> sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=ga_IE,
> timezone=America/Argentina/Jujuy
>    [junit4]   2> NOTE: Linux 3.8.0-41-generic amd64/Oracle Corporation
> 1.8.0_20-ea (64-bit)/cpus=8,threads=1,free=150337080,total=224395264
>    [junit4]   2> NOTE: All tests run in this JVM: [TestBBoxStrategy,
> DateNRStrategyTest]
>    [junit4] Completed on J1 in 8.88s, 42 tests, 1 failure, 10 skipped <<<
> FAILURES!
>
> [...truncated 31 lines...]
> BUILD FAILED
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The
> following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The
> following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The
> following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37:
> The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/build.xml:543:
> The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:2017:
> The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/module-build.xml:60:
> The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296:
> The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920:
> There were test failures: 17 suites, 125 tests, 1 failure, 12 ignored (2
> assumptions)
>
> Total time: 30 minutes 1 second
> Build step 'Invoke Ant' marked build as failure
> Description set: Java: 64bit/jdk1.8.0_20-ea-b15 -XX:-UseCompressedOops
> -XX:+UseParallelGC
> Archiving artifacts
> Recording test results
> Email was triggered for: Failure - Any
> Sending email for trigger: Failure - Any
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>

[JENKINS] Lucene-Solr-trunk-Linux (64bit/jdk1.8.0_20-ea-b15) - Build # 10516 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10516/
Java: 64bit/jdk1.8.0_20-ea-b15 -XX:-UseCompressedOops -XX:+UseParallelGC

1 tests failed.
FAILED:  org.apache.lucene.spatial.prefix.DateNRStrategyTest.testWithin {#4 seed=[3F4202D795A1E146:B3DDDC492CA0D99]}

Error Message:
Shouldn't match I#2:[-264000 TO -264000-11-20] Q:-264000

Stack Trace:
java.lang.AssertionError: Shouldn't match I#2:[-264000 TO -264000-11-20] Q:-264000
	at __randomizedtesting.SeedInfo.seed([3F4202D795A1E146:B3DDDC492CA0D99]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.lucene.spatial.prefix.BaseNonFuzzySpatialOpStrategyTest.fail(BaseNonFuzzySpatialOpStrategyTest.java:128)
	at org.apache.lucene.spatial.prefix.BaseNonFuzzySpatialOpStrategyTest.testOperation(BaseNonFuzzySpatialOpStrategyTest.java:117)
	at org.apache.lucene.spatial.prefix.BaseNonFuzzySpatialOpStrategyTest.testOperationRandomShapes(BaseNonFuzzySpatialOpStrategyTest.java:64)
	at org.apache.lucene.spatial.prefix.DateNRStrategyTest.testWithin(DateNRStrategyTest.java:59)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9391 lines...]
   [junit4] Suite: org.apache.lucene.spatial.prefix.DateNRStrategyTest
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DateNRStrategyTest -Dtests.method=testWithin -Dtests.seed=3F4202D795A1E146 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ga_IE -Dtests.timezone=America/Argentina/Jujuy -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 0.28s J1 | DateNRStrategyTest.testWithin {#4 seed=[3F4202D795A1E146:B3DDDC492CA0D99]} <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Shouldn't match I#2:[-264000 TO -264000-11-20] Q:-264000
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3F4202D795A1E146:B3DDDC492CA0D99]:0)
   [junit4]    > 	at org.apache.lucene.spatial.prefix.BaseNonFuzzySpatialOpStrategyTest.fail(BaseNonFuzzySpatialOpStrategyTest.java:128)
   [junit4]    > 	at org.apache.lucene.spatial.prefix.BaseNonFuzzySpatialOpStrategyTest.testOperation(BaseNonFuzzySpatialOpStrategyTest.java:117)
   [junit4]    > 	at org.apache.lucene.spatial.prefix.BaseNonFuzzySpatialOpStrategyTest.testOperationRandomShapes(BaseNonFuzzySpatialOpStrategyTest.java:64)
   [junit4]    > 	at org.apache.lucene.spatial.prefix.DateNRStrategyTest.testWithin(DateNRStrategyTest.java:59)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/build/spatial/test/J1/./temp/lucene.spatial.prefix.DateNRStrategyTest-3F4202D795A1E146-001
   [junit4]   2> NOTE: test params are: codec=Lucene40, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=ga_IE, timezone=America/Argentina/Jujuy
   [junit4]   2> NOTE: Linux 3.8.0-41-generic amd64/Oracle Corporation 1.8.0_20-ea (64-bit)/cpus=8,threads=1,free=150337080,total=224395264
   [junit4]   2> NOTE: All tests run in this JVM: [TestBBoxStrategy, DateNRStrategyTest]
   [junit4] Completed on J1 in 8.88s, 42 tests, 1 failure, 10 skipped <<< FAILURES!

[...truncated 31 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/build.xml:543: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:2017: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/module-build.xml:60: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 17 suites, 125 tests, 1 failure, 12 ignored (2 assumptions)

Total time: 30 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.8.0_20-ea-b15 -XX:-UseCompressedOops -XX:+UseParallelGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (64bit/jdk1.8.0_05) - Build # 10515 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10515/
Java: 64bit/jdk1.8.0_05 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

2 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
shard2 is not consistent.  Got 513 from http://127.0.0.1:49972/collection1lastClient and got 515 from http://127.0.0.1:54538/collection1

Stack Trace:
java.lang.AssertionError: shard2 is not consistent.  Got 513 from http://127.0.0.1:49972/collection1lastClient and got 515 from http://127.0.0.1:54538/collection1
	at __randomizedtesting.SeedInfo.seed([96A231B0F93D96E2:1744BFA88E62F6DE]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1131)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1110)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:150)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.SyncSliceTest.testDistribSearch

Error Message:
No live SolrServers available to handle this request

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
	at __randomizedtesting.SeedInfo.seed([96A231B0F93D96E2:1744BFA88E62F6DE]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:317)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:650)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1141)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1110)
	at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:236)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10678 lines...]
   [junit4] Suite: org.apache.solr.cloud.SyncSliceTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/init-core-data-001
   [junit4]   2> 75250 T210 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 75251 T210 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 75254 T210 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 75254 T210 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 75255 T211 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 75355 T210 oasc.ZkTestServer.run start zk server on port:47477
   [junit4]   2> 75356 T210 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 75469 T217 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6afce936 name:ZooKeeperConnection Watcher:127.0.0.1:47477 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 75469 T210 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 75469 T210 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 75474 T210 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 75476 T219 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c7cdc26 name:ZooKeeperConnection Watcher:127.0.0.1:47477/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 75476 T210 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 75477 T210 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 75480 T210 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 75483 T210 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 75489 T210 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 75492 T210 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 75495 T210 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 75503 T210 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 75504 T210 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 75506 T210 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 75507 T210 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 75509 T210 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 75509 T210 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 75511 T210 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 75512 T210 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 75513 T210 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 75514 T210 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 75515 T210 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 75516 T210 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 75518 T210 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 75518 T210 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 75520 T210 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 75520 T210 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 75522 T210 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 75523 T210 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 75524 T210 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 75525 T210 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 75531 T210 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 75536 T221 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7de31d54 name:ZooKeeperConnection Watcher:127.0.0.1:47477/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 75537 T210 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 75851 T210 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 76094 T210 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 76101 T210 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:53725
   [junit4]   2> 76105 T210 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 76105 T210 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 76106 T210 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-002
   [junit4]   2> 76106 T210 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-002/'
   [junit4]   2> 76123 T210 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-002/solr.xml
   [junit4]   2> 76154 T210 oasc.CoreContainer.<init> New CoreContainer 352180056
   [junit4]   2> 76154 T210 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-002/]
   [junit4]   2> 76155 T210 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 76155 T210 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 76156 T210 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 76156 T210 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 76156 T210 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 76157 T210 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 76157 T210 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 76157 T210 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 76157 T210 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 76160 T210 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 76161 T210 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 76161 T210 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 76161 T210 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:47477/solr
   [junit4]   2> 76162 T210 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 76163 T210 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 76173 T232 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74a0c31d name:ZooKeeperConnection Watcher:127.0.0.1:47477 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 76174 T210 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 76185 T210 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 76187 T234 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4847bd07 name:ZooKeeperConnection Watcher:127.0.0.1:47477/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 76188 T210 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 76189 T210 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 76193 T210 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 76196 T210 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 76203 T210 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 76206 T210 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 76209 T210 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 76219 T210 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53725_
   [junit4]   2> 76221 T210 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53725_
   [junit4]   2> 76232 T210 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 76234 T210 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 76240 T210 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:53725_
   [junit4]   2> 76241 T210 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 76243 T210 oasc.Overseer.start Overseer (id=91902198886825988-127.0.0.1:53725_-n_0000000000) starting
   [junit4]   2> 76248 T210 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 76259 T236 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 76259 T210 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 76266 T210 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 76269 T210 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 76274 T237 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 76274 T237 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 76278 T237 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 76283 T235 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 76285 T235 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 76285 T235 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:53725",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:53725_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 76285 T235 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 76286 T235 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 76288 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 76289 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 77278 T237 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 77279 T237 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 77279 T237 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 77279 T237 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 77280 T237 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 77280 T237 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-002/collection1/'
   [junit4]   2> 77281 T237 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 77281 T237 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 77281 T237 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 77299 T237 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 77307 T237 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 77308 T237 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 77312 T237 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 77394 T237 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 77398 T237 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 77399 T237 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 77408 T237 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 77411 T237 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 77413 T237 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 77414 T237 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 77414 T237 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 77414 T237 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 77415 T237 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 77415 T237 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 77415 T237 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 77415 T237 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 77416 T237 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-001/control/data/
   [junit4]   2> 77416 T237 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@40629871
   [junit4]   2> 77416 T237 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-001/control/data
   [junit4]   2> 77416 T237 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-001/control/data/index/
   [junit4]   2> 77417 T237 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 77417 T237 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-001/control/data/index
   [junit4]   2> 77417 T237 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=0.505859375, floorSegmentMB=1.1787109375, forceMergeDeletesPctAllowed=23.873940951367725, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8562724387992168
   [junit4]   2> 77418 T237 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@65430b48 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7ac048c0),segFN=segments_1,generation=1}
   [junit4]   2> 77418 T237 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 77420 T237 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 77420 T237 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 77421 T237 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 77421 T237 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 77421 T237 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 77421 T237 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 77421 T237 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 77422 T237 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 77422 T237 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 77422 T237 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 77422 T237 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 77423 T237 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 77423 T237 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 77423 T237 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 77423 T237 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 77423 T237 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 77424 T237 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 77425 T237 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 77427 T237 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 77427 T237 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 77427 T237 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=91.916015625, floorSegmentMB=1.05859375, forceMergeDeletesPctAllowed=11.790924324113863, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 77428 T237 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@65430b48 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7ac048c0),segFN=segments_1,generation=1}
   [junit4]   2> 77428 T237 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 77428 T237 oass.SolrIndexSearcher.<init> Opening Searcher@565e2c61[collection1] main
   [junit4]   2> 77428 T237 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 77429 T237 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 77429 T237 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 77430 T237 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 77430 T237 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 77430 T237 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 77430 T237 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 77431 T237 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 77431 T237 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 77431 T237 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 77434 T237 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 77435 T238 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@565e2c61[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 77435 T241 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:53725 collection:control_collection shard:shard1
   [junit4]   2> 77436 T210 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 77436 T210 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 77436 T241 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 77440 T210 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 77442 T241 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 77442 T243 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f5c0da6 name:ZooKeeperConnection Watcher:127.0.0.1:47477/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 77443 T210 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 77444 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77444 T241 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 77444 T241 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C25 name=collection1 org.apache.solr.core.SolrCore@1dc6e8ae url=https://127.0.0.1:53725/collection1 node=127.0.0.1:53725_ C25_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:53725, core=collection1, node_name=127.0.0.1:53725_}
   [junit4]   2> 77444 T241 C25 P53725 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:53725/collection1/
   [junit4]   2> 77445 T235 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 77444 T210 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 77445 T241 C25 P53725 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 77445 T241 C25 P53725 oasc.SyncStrategy.syncToMe https://127.0.0.1:53725/collection1/ has no replicas
   [junit4]   2> 77446 T241 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:53725/collection1/ shard1
   [junit4]   2> 77446 T241 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 77448 T210 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 77450 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77451 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77453 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77555 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 77555 T243 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 77602 T241 oasc.ZkController.register We are https://127.0.0.1:53725/collection1/ and leader is https://127.0.0.1:53725/collection1/
   [junit4]   2> 77603 T241 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:53725
   [junit4]   2> 77603 T241 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 77603 T241 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 77603 T241 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 77604 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77605 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77605 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77605 T241 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 77606 T235 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 77607 T235 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:53725",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:53725_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 77608 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77678 T210 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 77679 T210 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 77684 T210 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 77687 T210 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:34253
   [junit4]   2> 77689 T210 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 77689 T210 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 77689 T210 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-003
   [junit4]   2> 77690 T210 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-003/'
   [junit4]   2> 77700 T210 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-003/solr.xml
   [junit4]   2> 77712 T243 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 77712 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 77715 T210 oasc.CoreContainer.<init> New CoreContainer 1956401431
   [junit4]   2> 77715 T210 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-003/]
   [junit4]   2> 77716 T210 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 77716 T210 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 77716 T210 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 77716 T210 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 77716 T210 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 77717 T210 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 77717 T210 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 77717 T210 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 77717 T210 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 77720 T210 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 77721 T210 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 77721 T210 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 77721 T210 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:47477/solr
   [junit4]   2> 77721 T210 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 77722 T210 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 77724 T254 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5f4df2b name:ZooKeeperConnection Watcher:127.0.0.1:47477 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 77724 T210 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 77731 T210 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 77737 T256 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@116313b5 name:ZooKeeperConnection Watcher:127.0.0.1:47477/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 77737 T210 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 77743 T210 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 78745 T210 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34253_
   [junit4]   2> 78746 T210 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:34253_
   [junit4]   2> 78754 T257 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 78755 T257 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 78756 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 78757 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 78757 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 78756 T257 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 78759 T235 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 78759 T235 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:34253",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34253_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 78759 T235 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
   [junit4]   2> 78760 T235 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 78763 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 78865 T243 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 78865 T256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 78865 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 79757 T257 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 79758 T257 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 79759 T257 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 79760 T257 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 79761 T257 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 79762 T257 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-003/collection1/'
   [junit4]   2> 79762 T257 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 79762 T257 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 79763 T257 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 79797 T257 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 79807 T257 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 79808 T257 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 79813 T257 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 79955 T257 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 79960 T257 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 79961 T257 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 79971 T257 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 79976 T257 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 79980 T257 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 79981 T257 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 79988 T257 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 79988 T257 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 79990 T257 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 79990 T257 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 79990 T257 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 79991 T257 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 79991 T257 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-001/jetty1/
   [junit4]   2> 79992 T257 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@40629871
   [junit4]   2> 79993 T257 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-001/jetty1
   [junit4]   2> 79993 T257 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-001/jetty1/index/
   [junit4]   2> 79993 T257 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 79993 T257 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-001/jetty1/index
   [junit4]   2> 79994 T257 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=0.505859375, floorSegmentMB=1.1787109375, forceMergeDeletesPctAllowed=23.873940951367725, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8562724387992168
   [junit4]   2> 79995 T257 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@65723a0a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5f63851d),segFN=segments_1,generation=1}
   [junit4]   2> 79996 T257 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 80002 T257 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 80003 T257 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 80003 T257 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 80003 T257 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 80004 T257 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 80004 T257 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 80004 T257 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 80005 T257 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 80005 T257 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 80006 T257 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 80006 T257 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 80006 T257 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 80007 T257 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 80007 T257 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 80007 T257 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 80008 T257 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 80008 T257 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 80010 T257 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 80015 T257 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 80016 T257 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 80017 T257 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=91.916015625, floorSegmentMB=1.05859375, forceMergeDeletesPctAllowed=11.790924324113863, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 80018 T257 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@65723a0a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5f63851d),segFN=segments_1,generation=1}
   [junit4]   2> 80018 T257 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 80018 T257 oass.SolrIndexSearcher.<init> Opening Searcher@7b9e894e[collection1] main
   [junit4]   2> 80018 T257 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 80020 T257 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 80020 T257 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 80020 T257 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 80021 T257 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 80021 T257 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 80022 T257 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 80022 T257 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 80022 T257 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 80022 T257 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 80025 T258 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7b9e894e[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 80027 T257 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 80027 T261 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:34253 collection:collection1 shard:shard1
   [junit4]   2> 80028 T210 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 80028 T210 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 80029 T261 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 80042 T261 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 80048 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80048 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80048 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80049 T235 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 80050 T261 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 80050 T261 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C26 name=collection1 org.apache.solr.core.SolrCore@45edcdac url=https://127.0.0.1:34253/collection1 node=127.0.0.1:34253_ C26_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:34253, core=collection1, node_name=127.0.0.1:34253_}
   [junit4]   2> 80050 T261 C26 P34253 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:34253/collection1/
   [junit4]   2> 80050 T261 C26 P34253 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 80051 T261 C26 P34253 oasc.SyncStrategy.syncToMe https://127.0.0.1:34253/collection1/ has no replicas
   [junit4]   2> 80051 T261 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:34253/collection1/ shard1
   [junit4]   2> 80051 T261 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 80052 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80058 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80065 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80167 T243 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 80167 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 80167 T256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 80175 T261 oasc.ZkController.register We are https://127.0.0.1:34253/collection1/ and leader is https://127.0.0.1:34253/collection1/
   [junit4]   2> 80175 T261 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:34253
   [junit4]   2> 80176 T261 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 80176 T261 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 80176 T261 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 80178 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80178 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80178 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80179 T261 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 80180 T235 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 80180 T235 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:34253",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34253_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 80182 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80284 T243 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 80284 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 80284 T256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 80300 T210 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 80301 T210 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 80308 T210 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 80311 T210 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:49240
   [junit4]   2> 80313 T210 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 80314 T210 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 80314 T210 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-004
   [junit4]   2> 80315 T210 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-004/'
   [junit4]   2> 80326 T210 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-004/solr.xml
   [junit4]   2> 80340 T210 oasc.CoreContainer.<init> New CoreContainer 375141446
   [junit4]   2> 80340 T210 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-004/]
   [junit4]   2> 80341 T210 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 80341 T210 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 80341 T210 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 80342 T210 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 80342 T210 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 80342 T210 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 80343 T210 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 80343 T210 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 80343 T210 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 80348 T210 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 80349 T210 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 80349 T210 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 80349 T210 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:47477/solr
   [junit4]   2> 80349 T210 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 80350 T210 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 80354 T272 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b691793 name:ZooKeeperConnection Watcher:127.0.0.1:47477 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 80363 T210 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 80366 T210 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 80368 T274 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@68a779de name:ZooKeeperConnection Watcher:127.0.0.1:47477/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 80369 T210 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 80380 T210 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 81386 T210 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49240_
   [junit4]   2> 81387 T210 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49240_
   [junit4]   2> 81394 T275 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 81394 T275 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 81395 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 81395 T275 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 81395 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 81396 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 81396 T235 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 81397 T235 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:49240",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49240_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 81397 T235 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=1
   [junit4]   2> 81397 T235 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 81399 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 81501 T243 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 81501 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 81501 T274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 81501 T256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 82395 T275 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 82396 T275 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 82396 T275 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 82397 T275 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 82397 T275 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 82398 T275 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-004/collection1/'
   [junit4]   2> 82398 T275 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 82399 T275 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 82399 T275 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 82426 T275 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 82437 T275 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 82439 T275 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 82445 T275 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 82574 T275 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 82579 T275 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 82581 T275 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 82591 T275 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 82594 T275 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 82597 T275 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 82598 T275 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 82599 T275 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 82599 T275 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 82600 T275 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 82600 T275 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 82601 T275 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 82601 T275 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 82601 T275 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-001/jetty2/
   [junit4]   2> 82602 T275 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@40629871
   [junit4]   2> 82603 T275 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-001/jetty2
   [junit4]   2> 82603 T275 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-001/jetty2/index/
   [junit4]   2> 82603 T275 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 82604 T275 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-001/jetty2/index
   [junit4]   2> 82605 T275 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=0.505859375, floorSegmentMB=1.1787109375, forceMergeDeletesPctAllowed=23.873940951367725, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8562724387992168
   [junit4]   2> 82606 T275 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2b1b7d5a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7a9ffde),segFN=segments_1,generation=1}
   [junit4]   2> 82606 T275 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 82610 T275 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 82610 T275 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 82611 T275 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 82611 T275 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 82611 T275 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 82611 T275 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 82612 T275 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 82612 T275 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 82612 T275 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 82613 T275 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 82613 T275 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 82613 T275 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 82614 T275 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 82614 T275 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 82615 T275 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 82615 T275 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 82615 T275 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 82618 T275 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 82620 T275 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 82620 T275 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 82621 T275 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=91.916015625, floorSegmentMB=1.05859375, forceMergeDeletesPctAllowed=11.790924324113863, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 82622 T275 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2b1b7d5a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7a9ffde),segFN=segments_1,generation=1}
   [junit4]   2> 82622 T275 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 82622 T275 oass.SolrIndexSearcher.<init> Opening Searcher@6f96c3c8[collection1] main
   [junit4]   2> 82622 T275 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 82623 T275 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 82624 T275 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 82624 T275 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 82624 T275 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 82625 T275 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 82625 T275 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 82626 T275 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 82626 T275 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 82626 T275 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 82631 T276 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6f96c3c8[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 82631 T275 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 82633 T279 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:49240 collection:collection1 shard:shard1
   [junit4]   2> 82633 T210 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 82633 T210 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 82635 T279 oasc.ZkController.register We are https://127.0.0.1:49240/collection1/ and leader is https://127.0.0.1:34253/collection1/
   [junit4]   2> 82635 T279 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:49240
   [junit4]   2> 82636 T279 oasc.ZkController.checkRecovery Core needs to recover:collection1
   [junit4]   2> 82636 T279 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
   [junit4]   2> ASYNC  NEW_CORE C27 name=collection1 org.apache.solr.core.SolrCore@7f5bf2ae url=https://127.0.0.1:49240/collection1 node=127.0.0.1:49240_ C27_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:49240, core=collection1, node_name=127.0.0.1:49240_}
   [junit4]   2> 82653 T280 C27 P49240 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
   [junit4]   2> 82653 T279 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 82654 T280 C27 P49240 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
   [junit4]   2> 82654 T280 C27 P49240 oasc.RecoveryStrategy.doRecovery Publishing state of core collection1 as recovering, leader is https://127.0.0.1:34253/collection1/ and I am https://127.0.0.1:49240/collection1/
   [junit4]   2> 82658 T280 C27 P49240 oasc.ZkController.publish publishing core=collection1 state=recovering collection=collection1
   [junit4]   2> 82659 T280 C27 P49240 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 82674 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 82675 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 82675 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 82676 T235 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 82676 T235 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:49240",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49240_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 82678 T280 C27 P49240 oasc.RecoveryStrategy.sendPrepRecoveryCmd Sending prep recovery command to https://127.0.0.1:34253; WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1%3A49240_&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true
   [junit4]   2> 82684 T234 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 82796 T256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 82796 T274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 82796 T243 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 82796 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 82833 T252 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 82834 T252 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 82835 T252 oasha.CoreAdminHandler.handleWaitForStateAction Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
   [junit4]   2> 82835 T252 oasha.CoreAdminHandler.handleWaitForStateAction In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:49240_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"state":"recovering","base_url":"https://127.0.0.1:49240","core":"collection1","node_name":"127.0.0.1:49240_"}
   [junit4]   2> 82836 T252 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 0 seconds.
   [junit4]   2> 82836 T252 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:49240_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=3 
   [junit4]   2> 82987 T210 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 82988 T210 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 82993 T210 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 82996 T210 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:51491
   [junit4]   2> 82998 T210 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 82999 T210 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 82999 T210 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-005
   [junit4]   2> 82999 T210 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-96A231B0F93D96E2-001/tempDir-005/'
   [junit4]   2> 83018 T210 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/

[...truncated too long message...]

orkspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-96A231B0F93D96E2-001/tempDir-001/jetty10;done=false>>]
   [junit4]   2> 272707 T1206 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-96A231B0F93D96E2-001/tempDir-001/jetty10
   [junit4]   2> 272707 T1206 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-96A231B0F93D96E2-001/tempDir-001/jetty10/index.20140609141956277 [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-96A231B0F93D96E2-001/tempDir-001/jetty10/index.20140609141956277;done=false>>]
   [junit4]   2> 272707 T1206 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-96A231B0F93D96E2-001/tempDir-001/jetty10/index.20140609141956277
   [junit4]   2> 272708 T1206 oasc.LeaderElector$ElectionWatcher.process WARN  java.lang.RuntimeException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer
   [junit4]   2> 	at org.apache.solr.cloud.Overseer.createOverseerNode(Overseer.java:1273)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer.getInQueue(Overseer.java:1224)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer.getInQueue(Overseer.java:1220)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:190)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$9.execute(SolrZkClient.java:312)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$9.execute(SolrZkClient.java:309)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:309)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer.createOverseerNode(Overseer.java:1264)
   [junit4]   2> 	... 24 more
   [junit4]   2> 
   [junit4]   2> 272708 T1206 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 10)
   [junit4]   2> 272708 T1206 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 272709 T1206 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 272709 T1206 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.method=testDistribSearch -Dtests.seed=96A231B0F93D96E2 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=lt -Dtests.timezone=Europe/Warsaw -Dtests.file.encoding=UTF-8
   [junit4] FAILURE  154s J1 | ChaosMonkeySafeLeaderTest.testDistribSearch <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: shard2 is not consistent.  Got 513 from http://127.0.0.1:49972/collection1lastClient and got 515 from http://127.0.0.1:54538/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([96A231B0F93D96E2:1744BFA88E62F6DE]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1131)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1110)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:150)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 272731 T358 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-96A231B0F93D96E2-001
   [junit4]   2> NOTE: test params are: codec=Lucene42, sim=DefaultSimilarity, locale=lt, timezone=Europe/Warsaw
   [junit4]   2> NOTE: Linux 3.8.0-41-generic amd64/Oracle Corporation 1.8.0_05 (64-bit)/cpus=8,threads=1,free=57479552,total=199266304
   [junit4]   2> NOTE: All tests run in this JVM: [ZkSolrClientTest, SSLMigrationTest, AtomicUpdatesTest, ChaosMonkeySafeLeaderTest]
   [junit4] Completed on J1 in 154.31s, 1 test, 1 failure <<< FAILURES!

[...truncated 1124 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 410 suites, 1692 tests, 1 error, 1 failure, 36 ignored (15 assumptions)

Total time: 64 minutes 41 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.8.0_05 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (64bit/jdk1.7.0_60) - Build # 10514 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10514/
Java: 64bit/jdk1.7.0_60 -XX:+UseCompressedOops -XX:+UseG1GC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.SyncSliceTest.testDistribSearch

Error Message:
No live SolrServers available to handle this request

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
	at __randomizedtesting.SeedInfo.seed([7733E4D74C9F7CFD:F6D56ACF3BC01CC1]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:317)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:650)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1141)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1110)
	at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:236)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
	at sun.reflect.GeneratedMethodAccessor41.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11215 lines...]
   [junit4] Suite: org.apache.solr.cloud.SyncSliceTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/init-core-data-001
   [junit4]   2> 926319 T4495 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 926320 T4495 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 926325 T4495 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 926325 T4495 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 926326 T4496 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 926426 T4495 oasc.ZkTestServer.run start zk server on port:51029
   [junit4]   2> 926427 T4495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 926430 T4502 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@717a960e name:ZooKeeperConnection Watcher:127.0.0.1:51029 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 926430 T4495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 926431 T4495 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 926434 T4495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 926435 T4504 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e987443 name:ZooKeeperConnection Watcher:127.0.0.1:51029/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 926435 T4495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 926436 T4495 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 926437 T4495 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 926439 T4495 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 926440 T4495 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 926441 T4495 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 926442 T4495 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 926444 T4495 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 926445 T4495 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 926447 T4495 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 926447 T4495 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 926449 T4495 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 926449 T4495 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 926451 T4495 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 926451 T4495 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 926453 T4495 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 926453 T4495 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 926454 T4495 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 926455 T4495 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 926457 T4495 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 926457 T4495 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 926459 T4495 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 926459 T4495 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 926461 T4495 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 926461 T4495 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 926462 T4495 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 926463 T4495 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 926464 T4495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 926466 T4506 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2cf7774a name:ZooKeeperConnection Watcher:127.0.0.1:51029/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 926466 T4495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 926676 T4495 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 926681 T4495 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 926703 T4495 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:40524
   [junit4]   2> 926706 T4495 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 926706 T4495 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 926707 T4495 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-002
   [junit4]   2> 926707 T4495 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-002/'
   [junit4]   2> 926726 T4495 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-002/solr.xml
   [junit4]   2> 926759 T4495 oasc.CoreContainer.<init> New CoreContainer 1969963656
   [junit4]   2> 926759 T4495 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-002/]
   [junit4]   2> 926760 T4495 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 926760 T4495 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 926760 T4495 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 926761 T4495 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 926761 T4495 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 926761 T4495 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 926761 T4495 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 926761 T4495 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 926762 T4495 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 926764 T4495 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 926764 T4495 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 926764 T4495 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 926764 T4495 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51029/solr
   [junit4]   2> 926765 T4495 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 926765 T4495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 926767 T4517 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@679d97ce name:ZooKeeperConnection Watcher:127.0.0.1:51029 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 926767 T4495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 926770 T4495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 926772 T4519 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e50266b name:ZooKeeperConnection Watcher:127.0.0.1:51029/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 926773 T4495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 926775 T4495 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 926780 T4495 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 926784 T4495 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 926789 T4495 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 926793 T4495 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 926798 T4495 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 926808 T4495 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40524_
   [junit4]   2> 926810 T4495 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:40524_
   [junit4]   2> 926815 T4495 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 926818 T4495 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 926823 T4495 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:40524_
   [junit4]   2> 926824 T4495 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 926827 T4495 oasc.Overseer.start Overseer (id=91901731441278980-127.0.0.1:40524_-n_0000000000) starting
   [junit4]   2> 926831 T4495 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 926846 T4521 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 926847 T4495 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 926851 T4495 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 926852 T4495 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 926854 T4520 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 926860 T4522 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 926860 T4522 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 926861 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 926861 T4522 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 926863 T4520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 926864 T4520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:40524",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:40524_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 926865 T4520 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 926865 T4520 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 926869 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 926869 T4519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 927862 T4522 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 927862 T4522 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 927863 T4522 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 927864 T4522 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 927865 T4522 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 927865 T4522 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-002/collection1/'
   [junit4]   2> 927866 T4522 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 927867 T4522 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 927867 T4522 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 927919 T4522 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 927944 T4522 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 927945 T4522 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 927952 T4522 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 928173 T4522 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 928177 T4522 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 928179 T4522 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 928191 T4522 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 928195 T4522 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 928200 T4522 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 928201 T4522 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 928202 T4522 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 928202 T4522 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 928203 T4522 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 928204 T4522 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 928204 T4522 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 928204 T4522 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 928205 T4522 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/control/data/
   [junit4]   2> 928205 T4522 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6835755d
   [junit4]   2> 928207 T4522 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/control/data
   [junit4]   2> 928207 T4522 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/control/data/index/
   [junit4]   2> 928208 T4522 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 928208 T4522 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/control/data/index
   [junit4]   2> 928209 T4522 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 928210 T4522 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@50197cb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@763b6ff4),segFN=segments_1,generation=1}
   [junit4]   2> 928210 T4522 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 928216 T4522 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 928217 T4522 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 928217 T4522 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 928218 T4522 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 928218 T4522 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 928218 T4522 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 928219 T4522 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 928220 T4522 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 928220 T4522 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 928221 T4522 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 928221 T4522 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 928222 T4522 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 928222 T4522 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 928223 T4522 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 928223 T4522 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 928224 T4522 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 928225 T4522 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 928229 T4522 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 928233 T4522 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 928233 T4522 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 928234 T4522 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 928235 T4522 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@50197cb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@763b6ff4),segFN=segments_1,generation=1}
   [junit4]   2> 928236 T4522 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 928236 T4522 oass.SolrIndexSearcher.<init> Opening Searcher@2a025739[collection1] main
   [junit4]   2> 928236 T4522 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 928238 T4522 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 928238 T4522 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 928239 T4522 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 928239 T4522 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 928239 T4522 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 928240 T4522 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 928240 T4522 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 928240 T4522 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 928241 T4522 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 928242 T4523 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2a025739[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 928243 T4522 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 928244 T4526 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:40524 collection:control_collection shard:shard1
   [junit4]   2> 928244 T4495 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 928245 T4495 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 928245 T4526 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 928248 T4495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 928249 T4528 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@607b4c05 name:ZooKeeperConnection Watcher:127.0.0.1:51029/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 928249 T4495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 928250 T4526 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 928250 T4495 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 928251 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 928252 T4526 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 928252 T4526 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C777 name=collection1 org.apache.solr.core.SolrCore@3c8e6566 url=https://127.0.0.1:40524/collection1 node=127.0.0.1:40524_ C777_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:40524, core=collection1, node_name=127.0.0.1:40524_}
   [junit4]   2> 928253 T4526 C777 P40524 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:40524/collection1/
   [junit4]   2> 928253 T4526 C777 P40524 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 928253 T4526 C777 P40524 oasc.SyncStrategy.syncToMe https://127.0.0.1:40524/collection1/ has no replicas
   [junit4]   2> 928254 T4526 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:40524/collection1/ shard1
   [junit4]   2> 928254 T4526 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 928255 T4495 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 928255 T4520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 928260 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 928262 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 928267 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 928370 T4528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 928370 T4519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 928413 T4526 oasc.ZkController.register We are https://127.0.0.1:40524/collection1/ and leader is https://127.0.0.1:40524/collection1/
   [junit4]   2> 928413 T4526 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:40524
   [junit4]   2> 928413 T4526 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 928414 T4526 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 928414 T4526 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 928415 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 928415 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 928415 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 928415 T4526 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 928416 T4520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 928417 T4520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:40524",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:40524_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 928418 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 928479 T4495 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 928480 T4495 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 928485 T4495 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 928487 T4495 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:56441
   [junit4]   2> 928490 T4495 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 928490 T4495 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 928490 T4495 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-003
   [junit4]   2> 928490 T4495 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-003/'
   [junit4]   2> 928506 T4495 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-003/solr.xml
   [junit4]   2> 928520 T4519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 928520 T4528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 928533 T4495 oasc.CoreContainer.<init> New CoreContainer 1915715664
   [junit4]   2> 928534 T4495 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-003/]
   [junit4]   2> 928534 T4495 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 928535 T4495 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 928535 T4495 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 928535 T4495 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 928535 T4495 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 928535 T4495 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 928536 T4495 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 928536 T4495 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 928536 T4495 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 928538 T4495 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 928538 T4495 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 928539 T4495 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 928539 T4495 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51029/solr
   [junit4]   2> 928539 T4495 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 928540 T4495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 928541 T4539 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7cfdeead name:ZooKeeperConnection Watcher:127.0.0.1:51029 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 928541 T4495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 928543 T4495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 928544 T4541 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64f22911 name:ZooKeeperConnection Watcher:127.0.0.1:51029/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 928545 T4495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 928553 T4495 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 929555 T4495 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56441_
   [junit4]   2> 929556 T4495 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:56441_
   [junit4]   2> 929578 T4542 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 929579 T4542 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 929580 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 929580 T4542 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 929580 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 929581 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 929583 T4520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 929583 T4520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:56441",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:56441_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 929584 T4520 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
   [junit4]   2> 929584 T4520 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 929587 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 929690 T4541 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 929690 T4519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 929690 T4528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 930581 T4542 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 930581 T4542 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 930582 T4542 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 930582 T4542 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 930584 T4542 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 930584 T4542 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-003/collection1/'
   [junit4]   2> 930586 T4542 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 930586 T4542 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 930587 T4542 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 930644 T4542 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 930667 T4542 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 930669 T4542 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 930675 T4542 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 930863 T4542 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 930868 T4542 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 930870 T4542 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 930881 T4542 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 930885 T4542 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 930888 T4542 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 930889 T4542 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 930890 T4542 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 930890 T4542 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 930891 T4542 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 930892 T4542 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 930892 T4542 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 930892 T4542 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 930893 T4542 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/jetty1/
   [junit4]   2> 930893 T4542 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6835755d
   [junit4]   2> 930894 T4542 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/jetty1
   [junit4]   2> 930894 T4542 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/jetty1/index/
   [junit4]   2> 930895 T4542 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 930895 T4542 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/jetty1/index
   [junit4]   2> 930895 T4542 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 930896 T4542 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@36b1afa3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@157824ca),segFN=segments_1,generation=1}
   [junit4]   2> 930896 T4542 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 930902 T4542 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 930902 T4542 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 930903 T4542 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 930903 T4542 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 930903 T4542 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 930903 T4542 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 930904 T4542 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 930904 T4542 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 930905 T4542 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 930905 T4542 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 930906 T4542 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 930906 T4542 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 930906 T4542 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 930907 T4542 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 930907 T4542 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 930908 T4542 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 930909 T4542 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 930912 T4542 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 930915 T4542 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 930915 T4542 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 930916 T4542 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 930917 T4542 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@36b1afa3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@157824ca),segFN=segments_1,generation=1}
   [junit4]   2> 930917 T4542 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 930918 T4542 oass.SolrIndexSearcher.<init> Opening Searcher@309504a9[collection1] main
   [junit4]   2> 930918 T4542 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 930920 T4542 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 930920 T4542 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 930921 T4542 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 930921 T4542 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 930921 T4542 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 930921 T4542 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 930921 T4542 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 930922 T4542 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 930922 T4542 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 930926 T4543 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@309504a9[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 930930 T4542 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 930931 T4546 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:56441 collection:collection1 shard:shard1
   [junit4]   2> 930932 T4495 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 930932 T4495 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 930932 T4546 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 930938 T4546 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 930940 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930940 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930940 T4546 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 930940 T4546 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 930940 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C778 name=collection1 org.apache.solr.core.SolrCore@609ba084 url=https://127.0.0.1:56441/collection1 node=127.0.0.1:56441_ C778_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:56441, core=collection1, node_name=127.0.0.1:56441_}
   [junit4]   2> 930940 T4546 C778 P56441 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:56441/collection1/
   [junit4]   2> 930941 T4520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 930941 T4546 C778 P56441 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 930941 T4546 C778 P56441 oasc.SyncStrategy.syncToMe https://127.0.0.1:56441/collection1/ has no replicas
   [junit4]   2> 930941 T4546 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:56441/collection1/ shard1
   [junit4]   2> 930941 T4546 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 930942 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930944 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930949 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 931050 T4519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 931051 T4528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 931051 T4541 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 931095 T4546 oasc.ZkController.register We are https://127.0.0.1:56441/collection1/ and leader is https://127.0.0.1:56441/collection1/
   [junit4]   2> 931095 T4546 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:56441
   [junit4]   2> 931095 T4546 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 931096 T4546 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 931096 T4546 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 931097 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 931097 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 931097 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 931098 T4546 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 931098 T4520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 931099 T4520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:56441",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:56441_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 931100 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 931149 T4495 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 931149 T4495 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 931153 T4495 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 931154 T4495 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:38661
   [junit4]   2> 931156 T4495 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 931157 T4495 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 931157 T4495 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-004
   [junit4]   2> 931157 T4495 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-004/'
   [junit4]   2> 931172 T4495 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-004/solr.xml
   [junit4]   2> 931209 T4528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 931209 T4541 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 931209 T4519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 931221 T4495 oasc.CoreContainer.<init> New CoreContainer 952781019
   [junit4]   2> 931222 T4495 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-004/]
   [junit4]   2> 931223 T4495 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 931223 T4495 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 931223 T4495 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 931223 T4495 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 931223 T4495 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 931224 T4495 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 931224 T4495 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 931224 T4495 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 931224 T4495 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 931227 T4495 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 931227 T4495 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 931227 T4495 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 931228 T4495 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51029/solr
   [junit4]   2> 931228 T4495 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 931229 T4495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 931230 T4557 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2cfa11c5 name:ZooKeeperConnection Watcher:127.0.0.1:51029 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 931230 T4495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 931232 T4495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 931233 T4559 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c7f0d4d name:ZooKeeperConnection Watcher:127.0.0.1:51029/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 931233 T4495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 931237 T4495 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 932239 T4495 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38661_
   [junit4]   2> 932241 T4495 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38661_
   [junit4]   2> 932256 T4560 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 932256 T4560 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 932258 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 932258 T4560 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 932258 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 932259 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 932260 T4520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 932261 T4520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:38661",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38661_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 932262 T4520 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=1
   [junit4]   2> 932262 T4520 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 932265 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 932367 T4559 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 932368 T4519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 932368 T4528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 932368 T4541 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 933259 T4560 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 933259 T4560 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 933260 T4560 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 933261 T4560 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 933262 T4560 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 933262 T4560 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-004/collection1/'
   [junit4]   2> 933264 T4560 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 933264 T4560 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 933265 T4560 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 933322 T4560 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 933344 T4560 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 933345 T4560 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 933350 T4560 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 933528 T4560 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 933532 T4560 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 933534 T4560 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 933541 T4560 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 933543 T4560 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 933545 T4560 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 933546 T4560 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 933546 T4560 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 933546 T4560 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 933547 T4560 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 933547 T4560 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 933547 T4560 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 933547 T4560 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 933548 T4560 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/jetty2/
   [junit4]   2> 933548 T4560 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6835755d
   [junit4]   2> 933549 T4560 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/jetty2
   [junit4]   2> 933549 T4560 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/jetty2/index/
   [junit4]   2> 933549 T4560 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 933549 T4560 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/jetty2/index
   [junit4]   2> 933550 T4560 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 933550 T4560 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5354654 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7f647b25),segFN=segments_1,generation=1}
   [junit4]   2> 933550 T4560 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 933555 T4560 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 933556 T4560 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 933556 T4560 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 933556 T4560 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 933557 T4560 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 933557 T4560 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 933557 T4560 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 933557 T4560 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 933558 T4560 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 933558 T4560 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 933559 T4560 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 933559 T4560 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 933559 T4560 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 933559 T4560 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 933560 T4560 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 933560 T4560 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 933561 T4560 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 933563 T4560 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 933565 T4560 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 933565 T4560 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 933565 T4560 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 933566 T4560 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5354654 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7f647b25),segFN=segments_1,generation=1}
   [junit4]   2> 933566 T4560 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 933566 T4560 oass.SolrIndexSearcher.<init> Opening Searcher@170d6fb1[collection1] main
   [junit4]   2> 933566 T4560 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 933567 T4560 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 933567 T4560 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 933568 T4560 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 933568 T4560 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 933568 T4560 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 933569 T4560 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 933569 T4560 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 933569 T4560 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 933569 T4560 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 933571 T4561 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@170d6fb1[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 933573 T4560 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 933574 T4564 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:38661 collection:collection1 shard:shard1
   [junit4]   2> 933574 T4495 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 933574 T4495 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 933576 T4564 oasc.ZkController.register We are https://127.0.0.1:38661/collection1/ and leader is https://127.0.0.1:56441/collection1/
   [junit4]   2> 933576 T4564 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:38661
   [junit4]   2> 933576 T4564 oasc.ZkController.checkRecovery Core needs to recover:collection1
   [junit4]   2> 933576 T4564 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
   [junit4]   2> ASYNC  NEW_CORE C779 name=collection1 org.apache.solr.core.SolrCore@633414aa url=https://127.0.0.1:38661/collection1 node=127.0.0.1:38661_ C779_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:38661, core=collection1, node_name=127.0.0.1:38661_}
   [junit4]   2> 933577 T4565 C779 P38661 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
   [junit4]   2> 933577 T4564 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 933578 T4565 C779 P38661 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
   [junit4]   2> 933578 T4565 C779 P38661 oasc.RecoveryStrategy.doRecovery Publishing state of core collection1 as recovering, leader is https://127.0.0.1:56441/collection1/ and I am https://127.0.0.1:38661/collection1/
   [junit4]   2> 933578 T4565 C779 P38661 oasc.ZkController.publish publishing core=collection1 state=recovering collection=collection1
   [junit4]   2> 933579 T4565 C779 P38661 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 933580 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933580 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933580 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933581 T4520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 933581 T4565 C779 P38661 oasc.RecoveryStrategy.sendPrepRecoveryCmd Sending prep recovery command to https://127.0.0.1:56441; WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1%3A38661_&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true
   [junit4]   2> 933582 T4520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:38661",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38661_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 933585 T4519 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933652 T4535 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 933653 T4535 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 933653 T4535 oasha.CoreAdminHandler.handleWaitForStateAction Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
   [junit4]   2> 933654 T4535 oasha.CoreAdminHandler.handleWaitForStateAction In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:38661_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"state":"down","base_url":"https://127.0.0.1:38661","core":"collection1","node_name":"127.0.0.1:38661_"}
   [junit4]   2> 933691 T4528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 933691 T4559 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 933691 T4541 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 933691 T4519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 933802 T4495 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 933802 T4495 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 933806 T4495 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 933807 T4495 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:48723
   [junit4]   2> 933809 T4495 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 933809 T4495 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 933810 T4495 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-005
   [junit4]   2> 933810 T4495 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-005/'
   [junit4]   2> 933824 T4495 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-005/solr.xml
   [junit4]   2> 933847 T4495 oasc.CoreContainer.<init> New CoreContainer 1418664475
   [junit4]   2> 933847 T4495 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-005/]
   [junit4]   2> 933848 T4495 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 933848 T4495 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 933848 T4495 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 933848 T4495 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 933849 T4495 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 933849 T4495 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 933849 T4495 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 933849 T4495 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 933850 T4495 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 933852 T4495 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 933852 T4495 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 933852 T4495 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 933852 T4495 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51029/solr
   [junit4]   2> 933853 T4495 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 933853 T4495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 933855 T4577 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@148c6c97 name:ZooKeeperConnection Watcher:127.0.0.1:51029 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 933855 T4495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 933858 T4495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 933860 T4579 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e2b2a9c name:ZooKeeperConnection Watcher:127.0.0.1:51029/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 933860 T4495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 933873 T4495 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 934654 T4535 oasha.CoreAdminHandler.handleWaitForStateAction In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:38661_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"state":"recovering","base_url":"https://127.0.0.1:38661","core":"collection1","node_name":"127.0.0.1:38661_"}
   [junit4]   2> 934655 T4535 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
   [junit4]   2> 934655 T4535 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={version=2&action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:38661_&wt=javabin&onlyIfLeaderActive=true&checkLive=true&coreNodeName=core_node2&onlyIfLeader=true&state=recovering} status=0 QTime=1003 
   [junit4]   2> 934876 T4495 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48723_
   [junit4]   2> 934878 T4495 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48723_
   [junit4]   2> 934893 T4580 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 934894 T4580 oasc.ZkController.publish 

[...truncated too long message...]

solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/jetty4/index [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/jetty4/index;done=false>>]
   [junit4]   2> 995779 T4599 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001/tempDir-001/jetty4/index
   [junit4]   2> 995779 T4599 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/queue/qn-
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:243)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:240)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:240)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.createData(DistributedQueue.java:346)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.offer(DistributedQueue.java:330)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:190)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 995781 T4599 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 995781 T4599 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 995781 T4599 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 995782 T4599 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.SyncSliceTest-7733E4D74C9F7CFD-001
   [junit4]   2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=38), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=38)), sim=DefaultSimilarity, locale=ko_KR, timezone=America/Nome
   [junit4]   2> NOTE: Linux 3.8.0-41-generic amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=8,threads=1,free=164318568,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [RegexBoostProcessorTest, CoreAdminHandlerTest, DistributedDebugComponentTest, DistribCursorPagingTest, TestReplicationHandler, UpdateParamsTest, BasicFunctionalityTest, TestRemoteStreaming, SchemaVersionSpecificBehaviorTest, ZkControllerTest, AsyncMigrateRouteKeyTest, TestExceedMaxTermLength, TestFunctionQuery, TestGroupingSearch, MigrateRouteKeyTest, JsonLoaderTest, TestCharFilters, CSVRequestHandlerTest, CursorPagingTest, TestLuceneMatchVersion, TestInfoStreamLogging, TestRandomDVFaceting, SliceStateUpdateTest, TestPseudoReturnFields, TestCustomSort, TestSolrIndexConfig, SolrXmlInZkTest, UUIDFieldTest, TestManagedResourceStorage, TermVectorComponentTest, DeleteShardTest, CacheHeaderTest, TestQueryUtils, LoggingHandlerTest, FieldMutatingUpdateProcessorTest, SynonymTokenizerTest, ParsingFieldUpdateProcessorsTest, DistributedQueryComponentCustomSortTest, SOLR749Test, StatelessScriptUpdateProcessorFactoryTest, RemoteQueryErrorTest, MBeansHandlerTest, TestReloadAndDeleteDocs, TestManagedSynonymFilterFactory, ClusterStateUpdateTest, TestNRTOpen, TestNonDefinedSimilarityFactory, CopyFieldTest, TestReversedWildcardFilterFactory, SpatialFilterTest, OverseerRolesTest, SoftAutoCommitTest, CircularListTest, DocValuesTest, BadComponentTest, ShardRoutingTest, RankQueryTest, AutoCommitTest, TestFieldTypeResource, SampleTest, BasicDistributedZkTest, UnloadDistributedZkTest, TestValueSourceCache, ResponseLogComponentTest, TestFaceting, ScriptEngineTest, DateRangeFieldTest, SolrInfoMBeanTest, CollectionsAPIAsyncDistributedZkTest, FileBasedSpellCheckerTest, QueryElevationComponentTest, HdfsUnloadDistributedZkTest, DistributedSuggestComponentTest, TestClassNameShortening, TestRestManager, TestBM25SimilarityFactory, TestSolrXml, TestConfig, TestStandardQParsers, MultiThreadedOCPTest, TestDefaultSimilarityFactory, DistributedSpellCheckComponentTest, MergeStrategyTest, DistributedTermsComponentTest, TestCoreContainer, TestAnalyzeInfixSuggestions, EchoParamsTest, PeerSyncTest, IndexSchemaTest, TestMaxScoreQueryParser, TestSearchPerf, TestExtendedDismaxParser, LeaderElectionTest, TestCollectionAPI, TestImplicitCoreProperties, DirectUpdateHandlerOptimizeTest, SyncSliceTest]
   [junit4] Completed on J0 in 69.63s, 1 test, 1 error <<< FAILURES!

[...truncated 630 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 410 suites, 1692 tests, 1 error, 36 ignored (15 assumptions)

Total time: 62 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0_60 -XX:+UseCompressedOops -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



Re: [JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.8.0_20-ea-b15) - Build # 10513 - Still Failing!

Posted by Anshum Gupta <an...@anshumgupta.net>.
Committed a fix.

On Mon, Jun 9, 2014 at 12:56 AM, Policeman Jenkins Server
<je...@thetaphi.de> wrote:
> Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10513/
> Java: 32bit/jdk1.8.0_20-ea-b15 -server -XX:+UseParallelGC
>
> 1 tests failed.
> FAILED:  org.apache.solr.cloud.MultiThreadedOCPTest.testDistribSearch
>
> Error Message:
> After invoking OVERSEERSTATUS, SplitShard task [2000] was still supposed to be in [running] but isn't.It is [completed]
>
> Stack Trace:
> java.lang.AssertionError: After invoking OVERSEERSTATUS, SplitShard task [2000] was still supposed to be in [running] but isn't.It is [completed]
>         at __randomizedtesting.SeedInfo.seed([876CCE9B20D6C54E:68A40835789A572]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.junit.Assert.assertTrue(Assert.java:43)
>         at org.apache.solr.cloud.MultiThreadedOCPTest.testLongAndShortRunningParallelApiCalls(MultiThreadedOCPTest.java:209)
>         at org.apache.solr.cloud.MultiThreadedOCPTest.doTest(MultiThreadedOCPTest.java:71)
>         at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:483)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
>         at java.lang.Thread.run(Thread.java:745)
>
>
>
>
> Build Log:
> [...truncated 11041 lines...]
>    [junit4] Suite: org.apache.solr.cloud.MultiThreadedOCPTest
>    [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/init-core-data-001
>    [junit4]   2> 350379 T2109 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
>    [junit4]   2> 350380 T2109 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
>    [junit4]   2> 350394 T2109 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
>    [junit4]   2> 350409 T2109 oasc.ZkTestServer.run STARTING ZK TEST SERVER
>    [junit4]   1> client port:0.0.0.0/0.0.0.0:0
>    [junit4]   2> 350430 T2110 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
>    [junit4]   2> 350518 T2109 oasc.ZkTestServer.run start zk server on port:51949
>    [junit4]   2> 350533 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 350642 T2116 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1798141 name:ZooKeeperConnection Watcher:127.0.0.1:51949 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 350642 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 350642 T2109 oascc.SolrZkClient.makePath makePath: /solr
>    [junit4]   2> 350653 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 350655 T2118 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e90b5a name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 350655 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 350656 T2109 oascc.SolrZkClient.makePath makePath: /collections/collection1
>    [junit4]   2> 350658 T2109 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
>    [junit4]   2> 350660 T2109 oascc.SolrZkClient.makePath makePath: /collections/control_collection
>    [junit4]   2> 350662 T2109 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
>    [junit4]   2> 350680 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
>    [junit4]   2> 350680 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
>    [junit4]   2> 350697 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
>    [junit4]   2> 350698 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
>    [junit4]   2> 350716 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>    [junit4]   2> 350717 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>    [junit4]   2> 350721 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
>    [junit4]   2> 350722 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
>    [junit4]   2> 350725 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
>    [junit4]   2> 350726 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
>    [junit4]   2> 350729 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
>    [junit4]   2> 350730 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
>    [junit4]   2> 350733 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
>    [junit4]   2> 350733 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
>    [junit4]   2> 350736 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
>    [junit4]   2> 350736 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
>    [junit4]   2> 350738 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
>    [junit4]   2> 350739 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
>    [junit4]   2> 350743 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
>    [junit4]   2> 350743 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
>    [junit4]   2> 350747 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
>    [junit4]   2> 350747 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
>    [junit4]   2> 351023 T2109 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 351027 T2109 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:60423
>    [junit4]   2> 351027 T2109 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>    [junit4]   2> 351028 T2109 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>    [junit4]   2> 351028 T2109 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002
>    [junit4]   2> 351028 T2109 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/'
>    [junit4]   2> 351039 T2109 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/solr.xml
>    [junit4]   2> 351047 T2109 oasc.CoreContainer.<init> New CoreContainer 13020690
>    [junit4]   2> 351047 T2109 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/]
>    [junit4]   2> 351048 T2109 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
>    [junit4]   2> 351048 T2109 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
>    [junit4]   2> 351048 T2109 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>    [junit4]   2> 351048 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>    [junit4]   2> 351049 T2109 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>    [junit4]   2> 351049 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>    [junit4]   2> 351049 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>    [junit4]   2> 351049 T2109 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>    [junit4]   2> 351050 T2109 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>    [junit4]   2> 351050 T2109 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 351050 T2109 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 351051 T2109 oasc.CoreContainer.load Host Name: 127.0.0.1
>    [junit4]   2> 351051 T2109 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51949/solr
>    [junit4]   2> 351051 T2109 oasc.ZkController.checkChrootPath zkHost includes chroot
>    [junit4]   2> 351052 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 351069 T2129 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@994f9f name:ZooKeeperConnection Watcher:127.0.0.1:51949 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 351070 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 351080 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 351081 T2131 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f4a805 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 351081 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 351084 T2109 oascc.SolrZkClient.makePath makePath: /overseer/queue
>    [junit4]   2> 351087 T2109 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
>    [junit4]   2> 351110 T2109 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
>    [junit4]   2> 351120 T2109 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
>    [junit4]   2> 351125 T2109 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
>    [junit4]   2> 351130 T2109 oascc.SolrZkClient.makePath makePath: /live_nodes
>    [junit4]   2> 351146 T2109 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60423_
>    [junit4]   2> 351147 T2109 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:60423_
>    [junit4]   2> 351149 T2109 oascc.SolrZkClient.makePath makePath: /overseer_elect
>    [junit4]   2> 351150 T2109 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
>    [junit4]   2> 351165 T2109 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:60423_
>    [junit4]   2> 351165 T2109 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
>    [junit4]   2> 351168 T2109 oasc.Overseer.start Overseer (id=91901056920322051-127.0.0.1:60423_-n_0000000000) starting
>    [junit4]   2> 351171 T2109 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
>    [junit4]   2> 351183 T2133 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
>    [junit4]   2> 351184 T2109 oascc.SolrZkClient.makePath makePath: /clusterstate.json
>    [junit4]   2> 351192 T2109 oascc.SolrZkClient.makePath makePath: /aliases.json
>    [junit4]   2> 351193 T2109 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
>    [junit4]   2> 351195 T2132 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
>    [junit4]   2> 351198 T2134 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
>    [junit4]   2> 351198 T2134 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
>    [junit4]   2> 351199 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 351199 T2134 oasc.ZkController.waitForCoreNodeName look for our core node name
>    [junit4]   2> 351200 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
>    [junit4]   2> 351200 T2132 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "base_url":"http://127.0.0.1:60423",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:60423_",
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "numShards":"1",
>    [junit4]   2>          "core_node_name":null}
>    [junit4]   2> 351201 T2132 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
>    [junit4]   2> 351201 T2132 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
>    [junit4]   2> 351204 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 351205 T2131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 352201 T2134 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
>    [junit4]   2> 352202 T2134 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
>    [junit4]   2> 352202 T2134 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>    [junit4]   2> 352202 T2134 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
>    [junit4]   2> 352203 T2134 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 352203 T2134 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/collection1/'
>    [junit4]   2> 352204 T2134 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/collection1/lib/.svn/' to classloader
>    [junit4]   2> 352204 T2134 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/collection1/lib/classes/' to classloader
>    [junit4]   2> 352205 T2134 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/collection1/lib/README' to classloader
>    [junit4]   2> 352239 T2134 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
>    [junit4]   2> 352253 T2134 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>    [junit4]   2> 352268 T2134 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>    [junit4]   2> 352274 T2134 oass.IndexSchema.readSchema [collection1] Schema name=test
>    [junit4]   2> 352374 T2134 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>    [junit4]   2> 352381 T2134 oass.IndexSchema.readSchema default search field in schema is text
>    [junit4]   2> 352382 T2134 oass.IndexSchema.readSchema unique key field: id
>    [junit4]   2> 352408 T2134 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>    [junit4]   2> 352410 T2134 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>    [junit4]   2> 352413 T2134 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>    [junit4]   2> 352415 T2134 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>    [junit4]   2> 352416 T2134 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>    [junit4]   2> 352416 T2134 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>    [junit4]   2> 352420 T2134 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>    [junit4]   2> 352421 T2134 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>    [junit4]   2> 352421 T2134 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
>    [junit4]   2> 352421 T2134 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>    [junit4]   2> 352421 T2134 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-001/control/data/
>    [junit4]   2> 352422 T2134 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@eebee1
>    [junit4]   2> 352423 T2134 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-001/control/data
>    [junit4]   2> 352424 T2134 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-001/control/data/index/
>    [junit4]   2> 352424 T2134 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
>    [junit4]   2> 352424 T2134 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-001/control/data/index
>    [junit4]   2> 352424 T2134 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7919802637305674]
>    [junit4]   2> 352426 T2134 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>    [junit4]   2>                commit{dir=MockDirectoryWrapper(RAMDirectory@1cb195c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1d1ce36),segFN=segments_1,generation=1}
>    [junit4]   2> 352426 T2134 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 352431 T2134 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>    [junit4]   2> 352432 T2134 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>    [junit4]   2> 352432 T2134 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>    [junit4]   2> 352432 T2134 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 352433 T2134 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 352433 T2134 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>    [junit4]   2> 352433 T2134 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 352433 T2134 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 352434 T2134 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>    [junit4]   2> 352434 T2134 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>    [junit4]   2> 352435 T2134 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>    [junit4]   2> 352435 T2134 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>    [junit4]   2> 352435 T2134 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>    [junit4]   2> 352436 T2134 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>    [junit4]   2> 352436 T2134 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
>    [junit4]   2> 352436 T2134 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
>    [junit4]   2> 352437 T2134 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
>    [junit4]   2> 352439 T2134 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 352441 T2134 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>    [junit4]   2> 352441 T2134 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>    [junit4]   2> 352442 T2134 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=28.703125, floorSegmentMB=0.94140625, forceMergeDeletesPctAllowed=15.153936044953278, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
>    [junit4]   2> 352445 T2134 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>    [junit4]   2>                commit{dir=MockDirectoryWrapper(RAMDirectory@1cb195c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1d1ce36),segFN=segments_1,generation=1}
>    [junit4]   2> 352446 T2134 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 352446 T2134 oass.SolrIndexSearcher.<init> Opening Searcher@194d266[collection1] main
>    [junit4]   2> 352447 T2134 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
>    [junit4]   2> 352447 T2134 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 352447 T2134 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
>    [junit4]   2> 352448 T2134 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
>    [junit4]   2> 352448 T2134 oasr.RestManager.init Initializing RestManager with initArgs: {}
>    [junit4]   2> 352448 T2134 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 352448 T2134 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
>    [junit4]   2> 352449 T2134 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 352449 T2134 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
>    [junit4]   2> 352449 T2134 oasr.RestManager.init Initializing 0 registered ManagedResources
>    [junit4]   2> 352450 T2135 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@194d266[collection1] main{UninvertingDirectoryReader()}
>    [junit4]   2> 352451 T2134 oasc.CoreContainer.registerCore registering core: collection1
>    [junit4]   2> 352453 T2138 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:60423 collection:control_collection shard:shard1
>    [junit4]   2> 352453 T2109 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
>    [junit4]   2> 352453 T2109 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>    [junit4]   2> 352455 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 352455 T2138 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
>    [junit4]   2> 352459 T2140 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f0d759 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 352460 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 352461 T2109 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
>    [junit4]   2> 352474 T2109 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
>    [junit4]   2> 352477 T2138 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
>    [junit4]   2> 352478 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 352479 T2138 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
>    [junit4]   2> 352479 T2138 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
>    [junit4]   2> 352479 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
>    [junit4]   2> ASYNC  NEW_CORE C335 name=collection1 org.apache.solr.core.SolrCore@1676b88 url=http://127.0.0.1:60423/collection1 node=127.0.0.1:60423_ C335_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:60423, core=collection1, node_name=127.0.0.1:60423_}
>    [junit4]   2> 352479 T2138 C335 P60423 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:60423/collection1/
>    [junit4]   2> 352480 T2138 C335 P60423 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
>    [junit4]   2> 352480 T2138 C335 P60423 oasc.SyncStrategy.syncToMe http://127.0.0.1:60423/collection1/ has no replicas
>    [junit4]   2> 352481 T2138 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:60423/collection1/ shard1
>    [junit4]   2> 352481 T2138 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
>    [junit4]   2> 352502 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 352514 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 352517 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 352620 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 352620 T2131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 352664 T2138 oasc.ZkController.register We are http://127.0.0.1:60423/collection1/ and leader is http://127.0.0.1:60423/collection1/
>    [junit4]   2> 352665 T2138 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:60423
>    [junit4]   2> 352665 T2138 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
>    [junit4]   2> 352665 T2138 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
>    [junit4]   2> 352665 T2138 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
>    [junit4]   2> 352679 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 352679 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 352679 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 352680 T2138 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
>    [junit4]   2> 352680 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
>    [junit4]   2> 352681 T2132 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "state":"active",
>    [junit4]   2>          "base_url":"http://127.0.0.1:60423",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:60423_",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "numShards":"2",
>    [junit4]   2>          "core_node_name":"core_node1"}
>    [junit4]   2> 352710 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 352816 T2109 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
>    [junit4]   2> 352817 T2109 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 352818 T2109 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:47611
>    [junit4]   2> 352819 T2109 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>    [junit4]   2> 352819 T2109 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>    [junit4]   2> 352819 T2109 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003
>    [junit4]   2> 352820 T2109 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/'
>    [junit4]   2> 352821 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 352821 T2131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 352832 T2109 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/solr.xml
>    [junit4]   2> 352839 T2109 oasc.CoreContainer.<init> New CoreContainer 20568719
>    [junit4]   2> 352839 T2109 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/]
>    [junit4]   2> 352840 T2109 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
>    [junit4]   2> 352840 T2109 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
>    [junit4]   2> 352840 T2109 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>    [junit4]   2> 352841 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>    [junit4]   2> 352841 T2109 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>    [junit4]   2> 352841 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>    [junit4]   2> 352842 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>    [junit4]   2> 352842 T2109 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>    [junit4]   2> 352842 T2109 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>    [junit4]   2> 352843 T2109 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 352843 T2109 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 352844 T2109 oasc.CoreContainer.load Host Name: 127.0.0.1
>    [junit4]   2> 352844 T2109 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51949/solr
>    [junit4]   2> 352845 T2109 oasc.ZkController.checkChrootPath zkHost includes chroot
>    [junit4]   2> 352846 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 352861 T2151 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1519d4f name:ZooKeeperConnection Watcher:127.0.0.1:51949 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 352862 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 352868 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 352868 T2153 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b7602 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 352868 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 352873 T2109 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
>    [junit4]   2> 353875 T2109 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47611_
>    [junit4]   2> 353876 T2109 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:47611_
>    [junit4]   2> 353881 T2154 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
>    [junit4]   2> 353882 T2154 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
>    [junit4]   2> 353883 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 353883 T2154 oasc.ZkController.waitForCoreNodeName look for our core node name
>    [junit4]   2> 353883 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 353883 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 353884 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
>    [junit4]   2> 353884 T2132 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "base_url":"http://127.0.0.1:47611",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:47611_",
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "collection":"collection1",
>    [junit4]   2>          "numShards":"2",
>    [junit4]   2>          "core_node_name":null}
>    [junit4]   2> 353884 T2132 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
>    [junit4]   2> 353884 T2132 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
>    [junit4]   2> 353886 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 353987 T2131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 353988 T2153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 353987 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 354883 T2154 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
>    [junit4]   2> 354883 T2154 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
>    [junit4]   2> 354884 T2154 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>    [junit4]   2> 354884 T2154 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
>    [junit4]   2> 354885 T2154 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 354885 T2154 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/'
>    [junit4]   2> 354886 T2154 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/lib/.svn/' to classloader
>    [junit4]   2> 354886 T2154 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/lib/classes/' to classloader
>    [junit4]   2> 354886 T2154 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/lib/README' to classloader
>    [junit4]   2> 354910 T2154 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
>    [junit4]   2> 354917 T2154 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>    [junit4]   2> 354918 T2154 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>    [junit4]   2> 354922 T2154 oass.IndexSchema.readSchema [collection1] Schema name=test
>    [junit4]   2> 354995 T2154 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>    [junit4]   2> 354999 T2154 oass.IndexSchema.readSchema default search field in schema is text
>    [junit4]   2> 355000 T2154 oass.IndexSchema.readSchema unique key field: id
>    [junit4]   2> 355007 T2154 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>    [junit4]   2> 355009 T2154 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>    [junit4]   2> 355010 T2154 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>    [junit4]   2> 355011 T2154 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>    [junit4]   2> 355011 T2154 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>    [junit4]   2> 355011 T2154 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>    [junit4]   2> 355012 T2154 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>    [junit4]   2> 355012 T2154 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>    [junit4]   2> 355012 T2154 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
>    [junit4]   2> 355013 T2154 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>    [junit4]   2> 355013 T2154 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/data/
>    [junit4]   2> 355013 T2154 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@eebee1
>    [junit4]   2> 355014 T2154 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/data
>    [junit4]   2> 355014 T2154 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/data/index/
>    [junit4]   2> 355014 T2154 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/data/index' doesn't exist. Creating new index...
>    [junit4]   2> 355014 T2154 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/data/index
>    [junit4]   2> 355014 T2154 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7919802637305674]
>    [junit4]   2> 355015 T2154 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>    [junit4]   2>                commit{dir=MockDirectoryWrapper(RAMDirectory@1f42922 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1440fdb),segFN=segments_1,generation=1}
>    [junit4]   2> 355015 T2154 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 355017 T2154 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>    [junit4]   2> 355017 T2154 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>    [junit4]   2> 355017 T2154 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>    [junit4]   2> 355018 T2154 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 355018 T2154 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 355018 T2154 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>    [junit4]   2> 355018 T2154 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 355018 T2154 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 355019 T2154 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>    [junit4]   2> 355019 T2154 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>    [junit4]   2> 355019 T2154 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>    [junit4]   2> 355019 T2154 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>    [junit4]   2> 355019 T2154 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>    [junit4]   2> 355020 T2154 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>    [junit4]   2> 355020 T2154 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
>    [junit4]   2> 355020 T2154 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
>    [junit4]   2> 355021 T2154 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
>    [junit4]   2> 355022 T2154 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 355023 T2154 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>    [junit4]   2> 355023 T2154 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>    [junit4]   2> 355024 T2154 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=28.703125, floorSegmentMB=0.94140625, forceMergeDeletesPctAllowed=15.153936044953278, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
>    [junit4]   2> 355024 T2154 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>    [junit4]   2>                commit{dir=MockDirectoryWrapper(RAMDirectory@1f42922 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1440fdb),segFN=segments_1,generation=1}
>    [junit4]   2> 355024 T2154 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 355024 T2154 oass.SolrIndexSearcher.<init> Opening Searcher@17000a2[collection1] main
>    [junit4]   2> 355025 T2154 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
>    [junit4]   2> 355025 T2154 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 355025 T2154 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
>    [junit4]   2> 355026 T2154 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
>    [junit4]   2> 355026 T2154 oasr.RestManager.init Initializing RestManager with initArgs: {}
>    [junit4]   2> 355026 T2154 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 355027 T2154 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
>    [junit4]   2> 355027 T2154 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 355027 T2154 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
>    [junit4]   2> 355027 T2154 oasr.RestManager.init Initializing 0 registered ManagedResources
>    [junit4]   2> 355036 T2155 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@17000a2[collection1] main{UninvertingDirectoryReader()}
>    [junit4]   2> 355037 T2154 oasc.CoreContainer.registerCore registering core: collection1
>    [junit4]   2> 355038 T2158 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:47611 collection:collection1 shard:shard2
>    [junit4]   2> 355038 T2109 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
>    [junit4]   2> 355039 T2109 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>    [junit4]   2> 355039 T2158 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
>    [junit4]   2> 355043 T2158 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
>    [junit4]   2> 355046 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 355046 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 355046 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 355050 T2158 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
>    [junit4]   2> 355050 T2158 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
>    [junit4]   2> 355050 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
>    [junit4]   2> ASYNC  NEW_CORE C336 name=collection1 org.apache.solr.core.SolrCore@1e1ec0a url=http://127.0.0.1:47611/collection1 node=127.0.0.1:47611_ C336_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:47611, core=collection1, node_name=127.0.0.1:47611_}
>    [junit4]   2> 355050 T2158 C336 P47611 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:47611/collection1/
>    [junit4]   2> 355051 T2158 C336 P47611 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
>    [junit4]   2> 355053 T2158 C336 P47611 oasc.SyncStrategy.syncToMe http://127.0.0.1:47611/collection1/ has no replicas
>    [junit4]   2> 355054 T2158 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:47611/collection1/ shard2
>    [junit4]   2> 355054 T2158 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
>    [junit4]   2> 355054 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 355057 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 355059 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 355163 T2131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 355163 T2153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 355163 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 355209 T2158 oasc.ZkController.register We are http://127.0.0.1:47611/collection1/ and leader is http://127.0.0.1:47611/collection1/
>    [junit4]   2> 355209 T2158 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:47611
>    [junit4]   2> 355210 T2158 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
>    [junit4]   2> 355210 T2158 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
>    [junit4]   2> 355210 T2158 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
>    [junit4]   2> 355212 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 355212 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 355212 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 355213 T2158 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
>    [junit4]   2> 355213 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
>    [junit4]   2> 355214 T2132 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "state":"active",
>    [junit4]   2>          "base_url":"http://127.0.0.1:47611",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:47611_",
>    [junit4]   2>          "shard":"shard2",
>    [junit4]   2>          "collection":"collection1",
>    [junit4]   2>          "numShards":"2",
>    [junit4]   2>          "core_node_name":"core_node1"}
>    [junit4]   2> 355215 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 355276 T2109 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
>    [junit4]   2> 355277 T2109 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 355279 T2109 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:42158
>    [junit4]   2> 355280 T2109 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>    [junit4]   2> 355280 T2109 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>    [junit4]   2> 355280 T2109 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004
>    [junit4]   2> 355281 T2109 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/'
>    [junit4]   2> 355291 T2109 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/solr.xml
>    [junit4]   2> 355297 T2109 oasc.CoreContainer.<init> New CoreContainer 216655
>    [junit4]   2> 355297 T2109 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/]
>    [junit4]   2> 355298 T2109 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
>    [junit4]   2> 355298 T2109 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
>    [junit4]   2> 355298 T2109 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>    [junit4]   2> 355299 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>    [junit4]   2> 355299 T2109 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>    [junit4]   2> 355299 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>    [junit4]   2> 355300 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>    [junit4]   2> 355300 T2109 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>    [junit4]   2> 355300 T2109 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>    [junit4]   2> 355301 T2109 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 355301 T2109 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 355301 T2109 oasc.CoreContainer.load Host Name: 127.0.0.1
>    [junit4]   2> 355302 T2109 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51949/solr
>    [junit4]   2> 355302 T2109 oasc.ZkController.checkChrootPath zkHost includes chroot
>    [junit4]   2> 355303 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 355304 T2169 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@112b6c3 name:ZooKeeperConnection Watcher:127.0.0.1:51949 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 355304 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 355306 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 355306 T2171 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14ff266 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 355306 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 355310 T2109 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
>    [junit4]   2> 355316 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 355316 T2153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 355316 T2171 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 355316 T2131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 356312 T2109 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42158_
>    [junit4]   2> 356314 T2109 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42158_
>    [junit4]   2> 356319 T2172 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
>    [junit4]   2> 356319 T2172 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
>    [junit4]   2> 356327 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 356327 T2172 oasc.ZkController.waitForCoreNodeName look for our core node name
>    [junit4]   2> 356327 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 356328 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 356328 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
>    [junit4]   2> 356329 T2132 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "base_url":"http://127.0.0.1:42158",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:42158_",
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "collection":"collection1",
>    [junit4]   2>          "numShards":"2",
>    [junit4]   2>          "core_node_name":null}
>    [junit4]   2> 356329 T2132 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
>    [junit4]   2> 356329 T2132 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
>    [junit4]   2> 356331 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 356433 T2131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
>    [junit4]   2> 356433 T2171 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
>    [junit4]   2> 356433 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
>    [junit4]   2> 356433 T2153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
>    [junit4]   2> 357327 T2172 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
>    [junit4]   2> 357328 T2172 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
>    [junit4]   2> 357328 T2172 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>    [junit4]   2> 357329 T2172 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
>    [junit4]   2> 357329 T2172 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 357329 T2172 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/'
>    [junit4]   2> 357330 T2172 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/lib/.svn/' to classloader
>    [junit4]   2> 357330 T2172 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/lib/classes/' to classloader
>    [junit4]   2> 357331 T2172 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/lib/README' to classloader
>    [junit4]   2> 357354 T2172 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
>    [junit4]   2> 357365 T2172 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>    [junit4]   2> 357366 T2172 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>    [junit4]   2> 357372 T2172 oass.IndexSchema.readSchema [collection1] Schema name=test
>    [junit4]   2> 357481 T2172 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>    [junit4]   2> 357486 T2172 oass.IndexSchema.readSchema default search field in schema is text
>    [junit4]   2> 357487 T2172 oass.IndexSchema.readSchema unique key field: id
>    [junit4]   2> 357499 T2172 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>    [junit4]   2> 357501 T2172 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>    [junit4]   2> 357503 T2172 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>    [junit4]   2> 357503 T2172 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>    [junit4]   2> 357504 T2172 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>    [junit4]   2> 357504 T2172 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>    [junit4]   2> 357506 T2172 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>    [junit4]   2> 357506 T2172 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>    [junit4]   2> 357506 T2172 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
>    [junit4]   2> 357506 T2172 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>    [junit4]   2> 357507 T2172 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/data/
>    [junit4]   2> 357507 T2172 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@eebee1
>    [junit4]   2> 357508 T2172 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/data
>    [junit4]   2> 357508 T2172 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/data/index/
>    [junit4]   2> 357508 T2172 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/data/index' doesn't exist. Creating new index...
>    [junit4]   2> 357508 T2172 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/data/index
>    [junit4]   2> 357509 T2172 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7919802637305674]
>    [junit4]   2> 357510 T2172 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>    [junit4]   2>                commit{dir=MockDirectoryWrapper(RAMDirectory@1593c84 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@264ec9),segFN=segments_1,generation=1}
>    [junit4]   2> 357510 T2172 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 357513 T2172 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>    [junit4]   2> 357513 T2172 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>    [junit4]   2> 357514 T2172 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>    [junit4]   2> 357514 T2172 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 357515 T2172 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 357515 T2172 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>    [junit4]   2> 357516 T2172 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 357516 T2172 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 357516 T2172 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>    [junit4]   2> 357517 T2172 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>    [junit4]   2> 357517 T2172 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>    [junit4]   2> 357517 T2172 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>    [junit4]   2> 357518 T2172 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>    [junit4]   2> 357518 T2172 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>    [junit4]   2> 357518 T2172 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
>    [junit4]   2> 357519 T2172 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
>    [junit4]   2> 357519 T2172 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
>    [junit4]   2> 357521 T2172 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 357523 T2172 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>    [junit4]   2> 357523 T2172 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>    [junit4]   2> 357524 T2172 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=28.703125, floorSegmentMB=0.94140625, forceMergeDeletesPctAllowed=15.153936044953278, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
>    [junit4]   2> 357524 T2172 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>    [junit4]   2>                commit{dir=MockDirectoryWrapper(RAMDirectory@1593c84 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@264ec9),segFN=segments_1,generation=1}
>    [junit4]   2> 357524 T2172 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 357525 T2172 oass.SolrIndexSearcher.<init> Opening Searcher@653284[collection1] main
>    [junit4]   2> 357525 T2172 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
>    [junit4]   2> 357526 T2172 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 357526 T2172 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
>    [junit4]   2> 357527 T2172 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
>    [junit4]   2> 357527 T2172 oasr.RestManager.init Initializing RestManager with initArgs: {}
>    [junit4]   2> 357527 T2172 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 357528 T2172 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
>    [junit4]   2> 357528 T2172 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 357528 T2172 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
>    [junit4]   2> 357528 T2172 oasr.RestManager.init Initializing 0 registered ManagedResources
>    [junit4]   2> 357531 T2173 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@653284[collection1] main{UninvertingDirectoryReader()}
>    [junit4]   2> 357532 T2172 oasc.CoreContainer.registerCore registering core: collection1
>    [junit4]   2> 357533 T2176 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:42158 collection:collection1 shard:shard1
>    [junit4]   2> 357533 T2109 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
>    [junit4]   2> 357533 T2176 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
>    [junit4]   2> 357533 T2109 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>    [junit4]   2> 357536 T2176 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
>    [junit4]   2> 357538 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 357538 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 357538 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 357538 T2176 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
>    [junit4]   2> 357538 T2176 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
>    [junit4]   2> ASYNC  NEW_CORE C337 name=collection1 org.apache.solr.core.SolrCore@e077c4 url=http://127.0.0.1:42158/collection1 node=127.0.0.1:42158_ C337_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:42158, core=collection1, node_name=127.0.0.1:42158_}
>    [junit4]   2> 357539 T2176 C337 P42158 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:42158/collection1/
>    [junit4]   2> 357539 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
>    [junit4]   2> 357539 T2176 C337 P42158 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
>    [junit4]   2> 357539 T2176 C337 P42158 oasc.SyncStrategy.syncToMe http://127.0.0.1:42158/collection1/ has no replicas
>    [junit4]   2> 357539 T2176 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:42158/collection1/ shard1
>    [junit4]   2> 357540 T2176 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
>    [junit4]   2> 357540 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 357542 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 357545 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 357803 T2109 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
>    [junit4]   2> 357804 T2109 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 357806 T2109 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:41109
>    [junit4]   2> 357807 T2109 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>    [junit4]   2> 357808 T2109 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>    [junit4]   2> 357808 T2109 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-005
>    [junit4]   2> 357808 T2109 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-005/'
>    [junit4]   2> 357818 T2109 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-005/solr.xml
>    [junit4]   2> 357824 T2109 oasc.CoreContainer.<init> New CoreContainer 24516345
>    [junit4]   2> 357825 T2109 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-005/]
>    [junit4]   2> 357825 T2109 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
>    [junit4]   2> 357825 T2109 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
>    [junit4]   2> 357826 T2109 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>    [junit4]   2> 357826 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>    [junit4]   2> 357826 T2109 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>    [junit4]   2> 357826 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>    [junit4]   2> 357827 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>    [junit4]   2> 357827 T2109 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>    [junit4]   2> 357827 T2109 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>    [junit4]   2> 357828 T2109 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 357828 T2109 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 357828 T2109 oasc.CoreContainer.load Host Name: 127.0.0.1
>    [junit4]   2> 357829 T2109 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51949/solr
>    [junit4]   2> 357829 T2109 oasc.ZkController.checkChrootPath zkHost includes chroot
>    [junit4]   2> 357830 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 363103 T2111 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1467f863c380004, likely client has closed socket
>    [junit4]   2>        at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
>    [junit4]   2>        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
>    [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2>
>    [junit4]   2> 363203 T2140 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f0d759 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:Disconnected type:None path:null path:null type:None
>    [junit4]   2> 363204 T2140 oascc.ConnectionManager.process zkClient has disconnected
>    [junit4]   2> 364753 T2140 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f0d759 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 369774 T2111 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1467f863c380006, likely client has closed socket
>    [junit4]   2>        at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
>    [junit4]   2>        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
>    [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2>
>    [junit4]   2> 369874 T2153 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b7602 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:Disconnected type:None path:null path:null type:None
>    [junit4]   2> 369875 T2153 oascc.ConnectionManager.process zkClient has disconnected
>    [junit4]   2> 370883 T2111 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1467f863c380008, likely client has closed socket
>    [junit4]   2>        at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
>    [junit4]   2>        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
>    [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2>
>    [junit4]   2> 370984 T2171 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14ff266 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:Disconnected type:None path:null path:null type:None
>    [junit4]   2> 370984 T2171 oascc.ConnectionManager.process zkClient has disconnected
>    [junit4]   2> 370984 T2111 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1467f863c380003, likely client has closed socket
>    [junit4]   2>        at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
>    [junit4]   2>        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
>    [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2>
>    [junit4]   2> 371050 T2153 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b7602 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 371084 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
>    [junit4]   2> 371084 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
>    [junit4]   2> 371084 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
>    [junit4]   2> 371084 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
>    [junit4]   2> 371084 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
>    [junit4]   2> 371084 T2131 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f4a805 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:Disconnected type:None path:null path:null type:None
>    [junit4]   2> 371084 T2131 oascc.ConnectionManager.process zkClient has disconnected
>    [junit4]   2> 371085 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
>    [junit4]   2> 371085 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
>    [junit4]   2> 371085 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
>    [junit4]   2> 371085 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
>    [junit4]   2> 371422 T2111 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1467f863c380004, likely client has closed socket
>    [junit4]   2>        at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
>    [junit4]   2>        at org.apache
>
> [...truncated too long message...]
>
> erException.java:51)
>    [junit4]   2>        at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
>    [junit4]   2>        at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
>    [junit4]   2>        at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
>    [junit4]   2>        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
>    [junit4]   2>        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
>    [junit4]   2>        at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
>    [junit4]   2>        at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
>    [junit4]   2>        at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
>    [junit4]   2>        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
>    [junit4]   2>        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
>    [junit4]   2>
>    [junit4]   2> ASYNC  NEW_CORE C441 name=collection1 org.apache.solr.core.SolrCore@1ed1f7d url=http://127.0.0.1:33972/collection1 node=127.0.0.1:33972_ C441_STATE=coll:collection1 core:collection1 props:{state=active, base_url=http://127.0.0.1:33972, core=collection1, node_name=127.0.0.1:33972_}
>    [junit4]   2> 602470 T2209 C441 P33972 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
>    [junit4]   2> 602471 T2209 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is shutdown
>    [junit4]   2> 602471 T2209 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1ed1f7d
>    [junit4]   2> 602475 T2209 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=371,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=371,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
>    [junit4]   2> 602475 T2209 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
>    [junit4]   2> 602475 T2209 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
>    [junit4]   2> 602475 T2209 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
>    [junit4]   2> 602475 T2209 C441 P33972 oasu.DirectUpdateHandler2.closeWriter Committing on IndexWriter close.
>    [junit4]   2> 602476 T2209 C441 P33972 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
>    [junit4]   2>                commit{dir=MockDirectoryWrapper(RAMDirectory@6e57ed lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@e9b4cd),segFN=segments_1,generation=1}
>    [junit4]   2>                commit{dir=MockDirectoryWrapper(RAMDirectory@6e57ed lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@e9b4cd),segFN=segments_2,generation=2}
>    [junit4]   2> 602476 T2209 C441 P33972 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
>    [junit4]   2> 602477 T2209 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
>    [junit4]   2> 602477 T2209 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
>    [junit4]   2> 602477 T2209 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-006/collection1/data [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-006/collection1/data;done=false>>]
>    [junit4]   2> 602478 T2209 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-006/collection1/data
>    [junit4]   2> 602478 T2209 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-006/collection1/data/index [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-006/collection1/data/index;done=false>>]
>    [junit4]   2> 602478 T2209 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-006/collection1/data/index
>    [junit4]   2> 602478 T2209 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
>    [junit4]   2>        at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
>    [junit4]   2>        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
>    [junit4]   2>        at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
>    [junit4]   2>        at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
>    [junit4]   2>        at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
>    [junit4]   2>        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
>    [junit4]   2>        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
>    [junit4]   2>        at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
>    [junit4]   2>        at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
>    [junit4]   2>        at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
>    [junit4]   2>        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
>    [junit4]   2>        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
>    [junit4]   2>
>    [junit4]   2> 602478 T2209 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
>    [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001
>    [junit4]   2> NOTE: test params are: codec=Lucene49: {_version_=Pulsing41(freqCutoff=10 minBlockSize=9 maxBlockSize=41), multiDefault=FSTOrd41, intDefault=Pulsing41(freqCutoff=10 minBlockSize=9 maxBlockSize=41), id=PostingsFormat(name=NestedPulsing), range_facet_l=Pulsing41(freqCutoff=10 minBlockSize=9 maxBlockSize=41), timestamp=Pulsing41(freqCutoff=10 minBlockSize=9 maxBlockSize=41)}, docValues:{timestamp=DocValuesFormat(name=Lucene49)}, sim=DefaultSimilarity, locale=nl, timezone=Antarctica/Troll
>    [junit4]   2> NOTE: Linux 3.8.0-41-generic i386/Oracle Corporation 1.8.0_20-ea (32-bit)/cpus=8,threads=1,free=144613768,total=341311488
>    [junit4]   2> NOTE: All tests run in this JVM: [DocumentAnalysisRequestHandlerTest, TestDocBasedVersionConstraints, TestBinaryResponseWriter, OverseerRolesTest, NoFacetTest, TestFreeTextSuggestions, SolrPluginUtilsTest, HighlighterTest, TestCollationField, SolrRequestParserTest, OutputWriterTest, TestQuerySenderNoQuery, TestNoOpRegenerator, TestWordDelimiterFilterFactory, TestSolr4Spatial, NotRequiredUniqueKeyTest, TestFuzzyAnalyzedSuggestions, ShardRoutingCustomTest, TestReplicationHandlerBackup, SSLMigrationTest, DeleteReplicaTest, TestSolrJ, TestRecoveryHdfs, IndexSchemaRuntimeFieldTest, SpellCheckCollatorTest, DefaultValueUpdateProcessorTest, SimplePostToolTest, TestBinaryField, StatsComponentTest, CurrencyFieldOpenExchangeTest, TestReload, UniqFieldsUpdateProcessorFactoryTest, TestStressReorder, NoCacheHeaderTest, FileUtilsTest, JSONWriterTest, TestCoreDiscovery, HdfsDirectoryTest, AtomicUpdatesTest, TestDefaultSearchFieldResource, SolrXmlInZkTest, TestMultiCoreConfBootstrap, TestDistribDocBasedVersion, TestStressLucene, TestSolrCoreProperties, AbstractAnalyticsStatsTest, TestChildDocTransformer, TestDynamicFieldResource, FieldAnalysisRequestHandlerTest, DocValuesMultiTest, TestFieldTypeCollectionResource, ConvertedLegacyTest, HdfsBasicDistributedZk2Test, XsltUpdateRequestHandlerTest, TestFastOutputStream, TestLRUCache, TriLevelCompositeIdRoutingTest, DistributedSuggestComponentTest, MultiThreadedOCPTest]
>    [junit4] Completed on J1 in 252.74s, 1 test, 1 failure <<< FAILURES!
>
> [...truncated 858 lines...]
> BUILD FAILED
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 410 suites, 1693 tests, 1 failure, 36 ignored (15 assumptions)
>
> Total time: 81 minutes 12 seconds
> Build step 'Invoke Ant' marked build as failure
> Description set: Java: 32bit/jdk1.8.0_20-ea-b15 -server -XX:+UseParallelGC
> Archiving artifacts
> Recording test results
> Email was triggered for: Failure - Any
> Sending email for trigger: Failure - Any
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org



-- 

Anshum Gupta
http://www.anshumgupta.net

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.8.0_20-ea-b15) - Build # 10513 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10513/
Java: 32bit/jdk1.8.0_20-ea-b15 -server -XX:+UseParallelGC

1 tests failed.
FAILED:  org.apache.solr.cloud.MultiThreadedOCPTest.testDistribSearch

Error Message:
After invoking OVERSEERSTATUS, SplitShard task [2000] was still supposed to be in [running] but isn't.It is [completed]

Stack Trace:
java.lang.AssertionError: After invoking OVERSEERSTATUS, SplitShard task [2000] was still supposed to be in [running] but isn't.It is [completed]
	at __randomizedtesting.SeedInfo.seed([876CCE9B20D6C54E:68A40835789A572]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.MultiThreadedOCPTest.testLongAndShortRunningParallelApiCalls(MultiThreadedOCPTest.java:209)
	at org.apache.solr.cloud.MultiThreadedOCPTest.doTest(MultiThreadedOCPTest.java:71)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11041 lines...]
   [junit4] Suite: org.apache.solr.cloud.MultiThreadedOCPTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/init-core-data-001
   [junit4]   2> 350379 T2109 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 350380 T2109 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 350394 T2109 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 350409 T2109 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 350430 T2110 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 350518 T2109 oasc.ZkTestServer.run start zk server on port:51949
   [junit4]   2> 350533 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 350642 T2116 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1798141 name:ZooKeeperConnection Watcher:127.0.0.1:51949 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 350642 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 350642 T2109 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 350653 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 350655 T2118 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e90b5a name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 350655 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 350656 T2109 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 350658 T2109 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 350660 T2109 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 350662 T2109 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 350680 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 350680 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 350697 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 350698 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 350716 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 350717 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 350721 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 350722 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 350725 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 350726 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 350729 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 350730 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 350733 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 350733 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 350736 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 350736 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 350738 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 350739 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 350743 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 350743 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 350747 T2109 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 350747 T2109 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 351023 T2109 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 351027 T2109 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:60423
   [junit4]   2> 351027 T2109 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 351028 T2109 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 351028 T2109 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002
   [junit4]   2> 351028 T2109 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/'
   [junit4]   2> 351039 T2109 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/solr.xml
   [junit4]   2> 351047 T2109 oasc.CoreContainer.<init> New CoreContainer 13020690
   [junit4]   2> 351047 T2109 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/]
   [junit4]   2> 351048 T2109 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 351048 T2109 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 351048 T2109 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 351048 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 351049 T2109 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 351049 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 351049 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 351049 T2109 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 351050 T2109 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 351050 T2109 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 351050 T2109 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 351051 T2109 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 351051 T2109 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51949/solr
   [junit4]   2> 351051 T2109 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 351052 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 351069 T2129 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@994f9f name:ZooKeeperConnection Watcher:127.0.0.1:51949 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 351070 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 351080 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 351081 T2131 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f4a805 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 351081 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 351084 T2109 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 351087 T2109 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 351110 T2109 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 351120 T2109 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 351125 T2109 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 351130 T2109 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 351146 T2109 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60423_
   [junit4]   2> 351147 T2109 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:60423_
   [junit4]   2> 351149 T2109 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 351150 T2109 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 351165 T2109 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:60423_
   [junit4]   2> 351165 T2109 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 351168 T2109 oasc.Overseer.start Overseer (id=91901056920322051-127.0.0.1:60423_-n_0000000000) starting
   [junit4]   2> 351171 T2109 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 351183 T2133 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 351184 T2109 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 351192 T2109 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 351193 T2109 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 351195 T2132 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 351198 T2134 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 351198 T2134 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 351199 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 351199 T2134 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 351200 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 351200 T2132 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60423",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:60423_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 351201 T2132 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 351201 T2132 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 351204 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 351205 T2131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 352201 T2134 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 352202 T2134 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 352202 T2134 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 352202 T2134 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 352203 T2134 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 352203 T2134 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/collection1/'
   [junit4]   2> 352204 T2134 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 352204 T2134 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 352205 T2134 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 352239 T2134 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 352253 T2134 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 352268 T2134 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 352274 T2134 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 352374 T2134 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 352381 T2134 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 352382 T2134 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 352408 T2134 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 352410 T2134 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 352413 T2134 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 352415 T2134 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 352416 T2134 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 352416 T2134 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 352420 T2134 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 352421 T2134 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 352421 T2134 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 352421 T2134 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 352421 T2134 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-001/control/data/
   [junit4]   2> 352422 T2134 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@eebee1
   [junit4]   2> 352423 T2134 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-001/control/data
   [junit4]   2> 352424 T2134 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-001/control/data/index/
   [junit4]   2> 352424 T2134 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 352424 T2134 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-001/control/data/index
   [junit4]   2> 352424 T2134 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7919802637305674]
   [junit4]   2> 352426 T2134 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1cb195c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1d1ce36),segFN=segments_1,generation=1}
   [junit4]   2> 352426 T2134 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 352431 T2134 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 352432 T2134 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 352432 T2134 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 352432 T2134 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 352433 T2134 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 352433 T2134 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 352433 T2134 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 352433 T2134 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 352434 T2134 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 352434 T2134 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 352435 T2134 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 352435 T2134 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 352435 T2134 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 352436 T2134 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 352436 T2134 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 352436 T2134 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 352437 T2134 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 352439 T2134 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 352441 T2134 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 352441 T2134 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 352442 T2134 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=28.703125, floorSegmentMB=0.94140625, forceMergeDeletesPctAllowed=15.153936044953278, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 352445 T2134 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1cb195c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1d1ce36),segFN=segments_1,generation=1}
   [junit4]   2> 352446 T2134 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 352446 T2134 oass.SolrIndexSearcher.<init> Opening Searcher@194d266[collection1] main
   [junit4]   2> 352447 T2134 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 352447 T2134 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 352447 T2134 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 352448 T2134 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 352448 T2134 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 352448 T2134 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 352448 T2134 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 352449 T2134 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 352449 T2134 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 352449 T2134 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 352450 T2135 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@194d266[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 352451 T2134 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 352453 T2138 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:60423 collection:control_collection shard:shard1
   [junit4]   2> 352453 T2109 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 352453 T2109 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 352455 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 352455 T2138 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 352459 T2140 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f0d759 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 352460 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 352461 T2109 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 352474 T2109 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 352477 T2138 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 352478 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 352479 T2138 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 352479 T2138 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 352479 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C335 name=collection1 org.apache.solr.core.SolrCore@1676b88 url=http://127.0.0.1:60423/collection1 node=127.0.0.1:60423_ C335_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:60423, core=collection1, node_name=127.0.0.1:60423_}
   [junit4]   2> 352479 T2138 C335 P60423 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:60423/collection1/
   [junit4]   2> 352480 T2138 C335 P60423 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 352480 T2138 C335 P60423 oasc.SyncStrategy.syncToMe http://127.0.0.1:60423/collection1/ has no replicas
   [junit4]   2> 352481 T2138 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:60423/collection1/ shard1
   [junit4]   2> 352481 T2138 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 352502 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 352514 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 352517 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 352620 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 352620 T2131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 352664 T2138 oasc.ZkController.register We are http://127.0.0.1:60423/collection1/ and leader is http://127.0.0.1:60423/collection1/
   [junit4]   2> 352665 T2138 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:60423
   [junit4]   2> 352665 T2138 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 352665 T2138 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 352665 T2138 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 352679 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 352679 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 352679 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 352680 T2138 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352680 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352681 T2132 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60423",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:60423_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 352710 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 352816 T2109 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 352817 T2109 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 352818 T2109 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:47611
   [junit4]   2> 352819 T2109 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 352819 T2109 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 352819 T2109 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003
   [junit4]   2> 352820 T2109 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/'
   [junit4]   2> 352821 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 352821 T2131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 352832 T2109 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/solr.xml
   [junit4]   2> 352839 T2109 oasc.CoreContainer.<init> New CoreContainer 20568719
   [junit4]   2> 352839 T2109 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/]
   [junit4]   2> 352840 T2109 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 352840 T2109 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 352840 T2109 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 352841 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 352841 T2109 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 352841 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 352842 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 352842 T2109 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 352842 T2109 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 352843 T2109 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 352843 T2109 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 352844 T2109 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 352844 T2109 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51949/solr
   [junit4]   2> 352845 T2109 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 352846 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 352861 T2151 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1519d4f name:ZooKeeperConnection Watcher:127.0.0.1:51949 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 352862 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 352868 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 352868 T2153 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b7602 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 352868 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 352873 T2109 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 353875 T2109 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47611_
   [junit4]   2> 353876 T2109 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:47611_
   [junit4]   2> 353881 T2154 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 353882 T2154 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 353883 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 353883 T2154 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 353883 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 353883 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 353884 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353884 T2132 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:47611",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:47611_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 353884 T2132 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 353884 T2132 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 353886 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 353987 T2131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 353988 T2153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 353987 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 354883 T2154 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 354883 T2154 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 354884 T2154 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 354884 T2154 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 354885 T2154 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 354885 T2154 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/'
   [junit4]   2> 354886 T2154 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 354886 T2154 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 354886 T2154 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 354910 T2154 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 354917 T2154 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 354918 T2154 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 354922 T2154 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 354995 T2154 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 354999 T2154 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 355000 T2154 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 355007 T2154 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 355009 T2154 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 355010 T2154 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 355011 T2154 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 355011 T2154 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 355011 T2154 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 355012 T2154 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 355012 T2154 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 355012 T2154 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 355013 T2154 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 355013 T2154 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/data/
   [junit4]   2> 355013 T2154 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@eebee1
   [junit4]   2> 355014 T2154 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/data
   [junit4]   2> 355014 T2154 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/data/index/
   [junit4]   2> 355014 T2154 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 355014 T2154 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-003/collection1/data/index
   [junit4]   2> 355014 T2154 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7919802637305674]
   [junit4]   2> 355015 T2154 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1f42922 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1440fdb),segFN=segments_1,generation=1}
   [junit4]   2> 355015 T2154 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 355017 T2154 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 355017 T2154 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 355017 T2154 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 355018 T2154 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 355018 T2154 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 355018 T2154 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 355018 T2154 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 355018 T2154 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 355019 T2154 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 355019 T2154 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 355019 T2154 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 355019 T2154 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 355019 T2154 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 355020 T2154 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 355020 T2154 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 355020 T2154 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 355021 T2154 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 355022 T2154 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 355023 T2154 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 355023 T2154 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 355024 T2154 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=28.703125, floorSegmentMB=0.94140625, forceMergeDeletesPctAllowed=15.153936044953278, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 355024 T2154 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1f42922 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1440fdb),segFN=segments_1,generation=1}
   [junit4]   2> 355024 T2154 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 355024 T2154 oass.SolrIndexSearcher.<init> Opening Searcher@17000a2[collection1] main
   [junit4]   2> 355025 T2154 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 355025 T2154 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 355025 T2154 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 355026 T2154 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 355026 T2154 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 355026 T2154 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 355027 T2154 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 355027 T2154 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 355027 T2154 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 355027 T2154 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 355036 T2155 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@17000a2[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 355037 T2154 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 355038 T2158 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:47611 collection:collection1 shard:shard2
   [junit4]   2> 355038 T2109 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 355039 T2109 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 355039 T2158 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 355043 T2158 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 355046 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 355046 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 355046 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 355050 T2158 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 355050 T2158 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 355050 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C336 name=collection1 org.apache.solr.core.SolrCore@1e1ec0a url=http://127.0.0.1:47611/collection1 node=127.0.0.1:47611_ C336_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:47611, core=collection1, node_name=127.0.0.1:47611_}
   [junit4]   2> 355050 T2158 C336 P47611 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:47611/collection1/
   [junit4]   2> 355051 T2158 C336 P47611 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 355053 T2158 C336 P47611 oasc.SyncStrategy.syncToMe http://127.0.0.1:47611/collection1/ has no replicas
   [junit4]   2> 355054 T2158 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:47611/collection1/ shard2
   [junit4]   2> 355054 T2158 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 355054 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 355057 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 355059 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 355163 T2131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 355163 T2153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 355163 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 355209 T2158 oasc.ZkController.register We are http://127.0.0.1:47611/collection1/ and leader is http://127.0.0.1:47611/collection1/
   [junit4]   2> 355209 T2158 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:47611
   [junit4]   2> 355210 T2158 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 355210 T2158 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 355210 T2158 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 355212 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 355212 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 355212 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 355213 T2158 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355213 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355214 T2132 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:47611",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:47611_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 355215 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 355276 T2109 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 355277 T2109 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 355279 T2109 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:42158
   [junit4]   2> 355280 T2109 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 355280 T2109 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 355280 T2109 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004
   [junit4]   2> 355281 T2109 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/'
   [junit4]   2> 355291 T2109 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/solr.xml
   [junit4]   2> 355297 T2109 oasc.CoreContainer.<init> New CoreContainer 216655
   [junit4]   2> 355297 T2109 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/]
   [junit4]   2> 355298 T2109 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 355298 T2109 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 355298 T2109 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 355299 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 355299 T2109 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 355299 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 355300 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 355300 T2109 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 355300 T2109 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 355301 T2109 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 355301 T2109 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 355301 T2109 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 355302 T2109 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51949/solr
   [junit4]   2> 355302 T2109 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 355303 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 355304 T2169 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@112b6c3 name:ZooKeeperConnection Watcher:127.0.0.1:51949 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 355304 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 355306 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 355306 T2171 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14ff266 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 355306 T2109 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 355310 T2109 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 355316 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 355316 T2153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 355316 T2171 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 355316 T2131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 356312 T2109 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42158_
   [junit4]   2> 356314 T2109 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42158_
   [junit4]   2> 356319 T2172 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 356319 T2172 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 356327 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 356327 T2172 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 356327 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 356328 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 356328 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356329 T2132 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42158",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42158_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 356329 T2132 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 356329 T2132 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 356331 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 356433 T2131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 356433 T2171 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 356433 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 356433 T2153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 357327 T2172 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 357328 T2172 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 357328 T2172 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 357329 T2172 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 357329 T2172 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 357329 T2172 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/'
   [junit4]   2> 357330 T2172 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 357330 T2172 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 357331 T2172 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 357354 T2172 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 357365 T2172 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 357366 T2172 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 357372 T2172 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 357481 T2172 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 357486 T2172 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 357487 T2172 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 357499 T2172 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 357501 T2172 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 357503 T2172 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 357503 T2172 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 357504 T2172 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 357504 T2172 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 357506 T2172 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 357506 T2172 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 357506 T2172 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 357506 T2172 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 357507 T2172 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/data/
   [junit4]   2> 357507 T2172 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@eebee1
   [junit4]   2> 357508 T2172 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/data
   [junit4]   2> 357508 T2172 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/data/index/
   [junit4]   2> 357508 T2172 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 357508 T2172 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-004/collection1/data/index
   [junit4]   2> 357509 T2172 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7919802637305674]
   [junit4]   2> 357510 T2172 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1593c84 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@264ec9),segFN=segments_1,generation=1}
   [junit4]   2> 357510 T2172 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 357513 T2172 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 357513 T2172 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 357514 T2172 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 357514 T2172 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 357515 T2172 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 357515 T2172 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 357516 T2172 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 357516 T2172 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 357516 T2172 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 357517 T2172 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 357517 T2172 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 357517 T2172 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 357518 T2172 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 357518 T2172 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 357518 T2172 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 357519 T2172 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 357519 T2172 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 357521 T2172 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 357523 T2172 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 357523 T2172 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 357524 T2172 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=28.703125, floorSegmentMB=0.94140625, forceMergeDeletesPctAllowed=15.153936044953278, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 357524 T2172 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1593c84 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@264ec9),segFN=segments_1,generation=1}
   [junit4]   2> 357524 T2172 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 357525 T2172 oass.SolrIndexSearcher.<init> Opening Searcher@653284[collection1] main
   [junit4]   2> 357525 T2172 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 357526 T2172 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 357526 T2172 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 357527 T2172 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 357527 T2172 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 357527 T2172 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 357528 T2172 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 357528 T2172 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 357528 T2172 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 357528 T2172 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 357531 T2173 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@653284[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 357532 T2172 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 357533 T2176 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:42158 collection:collection1 shard:shard1
   [junit4]   2> 357533 T2109 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 357533 T2176 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 357533 T2109 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 357536 T2176 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 357538 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 357538 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 357538 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 357538 T2176 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 357538 T2176 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C337 name=collection1 org.apache.solr.core.SolrCore@e077c4 url=http://127.0.0.1:42158/collection1 node=127.0.0.1:42158_ C337_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:42158, core=collection1, node_name=127.0.0.1:42158_}
   [junit4]   2> 357539 T2176 C337 P42158 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:42158/collection1/
   [junit4]   2> 357539 T2132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 357539 T2176 C337 P42158 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 357539 T2176 C337 P42158 oasc.SyncStrategy.syncToMe http://127.0.0.1:42158/collection1/ has no replicas
   [junit4]   2> 357539 T2176 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:42158/collection1/ shard1
   [junit4]   2> 357540 T2176 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 357540 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 357542 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 357545 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 357803 T2109 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 357804 T2109 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 357806 T2109 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:41109
   [junit4]   2> 357807 T2109 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 357808 T2109 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 357808 T2109 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-005
   [junit4]   2> 357808 T2109 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-005/'
   [junit4]   2> 357818 T2109 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-005/solr.xml
   [junit4]   2> 357824 T2109 oasc.CoreContainer.<init> New CoreContainer 24516345
   [junit4]   2> 357825 T2109 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-005/]
   [junit4]   2> 357825 T2109 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 357825 T2109 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 357826 T2109 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 357826 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 357826 T2109 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 357826 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 357827 T2109 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 357827 T2109 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 357827 T2109 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 357828 T2109 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 357828 T2109 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 357828 T2109 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 357829 T2109 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51949/solr
   [junit4]   2> 357829 T2109 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 357830 T2109 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 363103 T2111 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1467f863c380004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 363203 T2140 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f0d759 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:Disconnected type:None path:null path:null type:None
   [junit4]   2> 363204 T2140 oascc.ConnectionManager.process zkClient has disconnected
   [junit4]   2> 364753 T2140 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f0d759 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 369774 T2111 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1467f863c380006, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 369874 T2153 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b7602 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:Disconnected type:None path:null path:null type:None
   [junit4]   2> 369875 T2153 oascc.ConnectionManager.process zkClient has disconnected
   [junit4]   2> 370883 T2111 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1467f863c380008, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 370984 T2171 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14ff266 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:Disconnected type:None path:null path:null type:None
   [junit4]   2> 370984 T2171 oascc.ConnectionManager.process zkClient has disconnected
   [junit4]   2> 370984 T2111 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1467f863c380003, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 371050 T2153 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b7602 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 371084 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
   [junit4]   2> 371084 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
   [junit4]   2> 371084 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
   [junit4]   2> 371084 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
   [junit4]   2> 371084 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
   [junit4]   2> 371084 T2131 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f4a805 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:Disconnected type:None path:null path:null type:None
   [junit4]   2> 371084 T2131 oascc.ConnectionManager.process zkClient has disconnected
   [junit4]   2> 371085 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
   [junit4]   2> 371085 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
   [junit4]   2> 371085 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
   [junit4]   2> 371085 T2131 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: null state: Disconnected type None
   [junit4]   2> 371422 T2111 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1467f863c380004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache

[...truncated too long message...]

erException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> ASYNC  NEW_CORE C441 name=collection1 org.apache.solr.core.SolrCore@1ed1f7d url=http://127.0.0.1:33972/collection1 node=127.0.0.1:33972_ C441_STATE=coll:collection1 core:collection1 props:{state=active, base_url=http://127.0.0.1:33972, core=collection1, node_name=127.0.0.1:33972_}
   [junit4]   2> 602470 T2209 C441 P33972 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
   [junit4]   2> 602471 T2209 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is shutdown
   [junit4]   2> 602471 T2209 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1ed1f7d
   [junit4]   2> 602475 T2209 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=371,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=371,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 602475 T2209 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 602475 T2209 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 602475 T2209 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 602475 T2209 C441 P33972 oasu.DirectUpdateHandler2.closeWriter Committing on IndexWriter close.
   [junit4]   2> 602476 T2209 C441 P33972 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6e57ed lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@e9b4cd),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6e57ed lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@e9b4cd),segFN=segments_2,generation=2}
   [junit4]   2> 602476 T2209 C441 P33972 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 602477 T2209 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 602477 T2209 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 602477 T2209 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-006/collection1/data [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-006/collection1/data;done=false>>]
   [junit4]   2> 602478 T2209 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-006/collection1/data
   [junit4]   2> 602478 T2209 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-006/collection1/data/index [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-006/collection1/data/index;done=false>>]
   [junit4]   2> 602478 T2209 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001/tempDir-006/collection1/data/index
   [junit4]   2> 602478 T2209 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 602478 T2209 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-876CCE9B20D6C54E-001
   [junit4]   2> NOTE: test params are: codec=Lucene49: {_version_=Pulsing41(freqCutoff=10 minBlockSize=9 maxBlockSize=41), multiDefault=FSTOrd41, intDefault=Pulsing41(freqCutoff=10 minBlockSize=9 maxBlockSize=41), id=PostingsFormat(name=NestedPulsing), range_facet_l=Pulsing41(freqCutoff=10 minBlockSize=9 maxBlockSize=41), timestamp=Pulsing41(freqCutoff=10 minBlockSize=9 maxBlockSize=41)}, docValues:{timestamp=DocValuesFormat(name=Lucene49)}, sim=DefaultSimilarity, locale=nl, timezone=Antarctica/Troll
   [junit4]   2> NOTE: Linux 3.8.0-41-generic i386/Oracle Corporation 1.8.0_20-ea (32-bit)/cpus=8,threads=1,free=144613768,total=341311488
   [junit4]   2> NOTE: All tests run in this JVM: [DocumentAnalysisRequestHandlerTest, TestDocBasedVersionConstraints, TestBinaryResponseWriter, OverseerRolesTest, NoFacetTest, TestFreeTextSuggestions, SolrPluginUtilsTest, HighlighterTest, TestCollationField, SolrRequestParserTest, OutputWriterTest, TestQuerySenderNoQuery, TestNoOpRegenerator, TestWordDelimiterFilterFactory, TestSolr4Spatial, NotRequiredUniqueKeyTest, TestFuzzyAnalyzedSuggestions, ShardRoutingCustomTest, TestReplicationHandlerBackup, SSLMigrationTest, DeleteReplicaTest, TestSolrJ, TestRecoveryHdfs, IndexSchemaRuntimeFieldTest, SpellCheckCollatorTest, DefaultValueUpdateProcessorTest, SimplePostToolTest, TestBinaryField, StatsComponentTest, CurrencyFieldOpenExchangeTest, TestReload, UniqFieldsUpdateProcessorFactoryTest, TestStressReorder, NoCacheHeaderTest, FileUtilsTest, JSONWriterTest, TestCoreDiscovery, HdfsDirectoryTest, AtomicUpdatesTest, TestDefaultSearchFieldResource, SolrXmlInZkTest, TestMultiCoreConfBootstrap, TestDistribDocBasedVersion, TestStressLucene, TestSolrCoreProperties, AbstractAnalyticsStatsTest, TestChildDocTransformer, TestDynamicFieldResource, FieldAnalysisRequestHandlerTest, DocValuesMultiTest, TestFieldTypeCollectionResource, ConvertedLegacyTest, HdfsBasicDistributedZk2Test, XsltUpdateRequestHandlerTest, TestFastOutputStream, TestLRUCache, TriLevelCompositeIdRoutingTest, DistributedSuggestComponentTest, MultiThreadedOCPTest]
   [junit4] Completed on J1 in 252.74s, 1 test, 1 failure <<< FAILURES!

[...truncated 858 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 410 suites, 1693 tests, 1 failure, 36 ignored (15 assumptions)

Total time: 81 minutes 12 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.8.0_20-ea-b15 -server -XX:+UseParallelGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (64bit/jdk1.7.0_60) - Build # 10512 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10512/
Java: 64bit/jdk1.7.0_60 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.MultiThreadedOCPTest.testDistribSearch

Error Message:
overseerstatus the collection error [Watcher fired on path: null state: SyncConnected type None]

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: overseerstatus the collection error [Watcher fired on path: null state: SyncConnected type None]
	at __randomizedtesting.SeedInfo.seed([DD151805B732301F:5CF3961DC06D5023]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:554)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.invokeCollectionApi(AbstractFullDistribZkTestBase.java:1738)
	at org.apache.solr.cloud.MultiThreadedOCPTest.testLongAndShortRunningParallelApiCalls(MultiThreadedOCPTest.java:195)
	at org.apache.solr.cloud.MultiThreadedOCPTest.doTest(MultiThreadedOCPTest.java:71)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
	at sun.reflect.GeneratedMethodAccessor49.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11594 lines...]
   [junit4] Suite: org.apache.solr.cloud.MultiThreadedOCPTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/init-core-data-001
   [junit4]   2> 1368574 T5636 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 1368575 T5636 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /z/
   [junit4]   2> 1368578 T5636 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 1368578 T5636 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1368579 T5637 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1368679 T5636 oasc.ZkTestServer.run start zk server on port:45432
   [junit4]   2> 1368680 T5636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1393181 T5640 oazsp.FileTxnLog.commit WARN fsync-ing the write ahead log in SyncThread:0 took 24499ms which will adversely effect operation latency. See the ZooKeeper troubleshooting guide
   [junit4]   2> 1393184 T5638 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1467f01a6e20000, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 1393185 T5638 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1467f01a6e20001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 1393186 T5643 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60f93e86 name:ZooKeeperConnection Watcher:127.0.0.1:45432 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1393187 T5636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1393187 T5636 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1393191 T5636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1393193 T5645 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@294bf6ca name:ZooKeeperConnection Watcher:127.0.0.1:45432/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1393193 T5636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1393194 T5636 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1393199 T5636 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1393202 T5636 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1393205 T5636 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1393208 T5636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1393208 T5636 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1393211 T5636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1393212 T5636 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1393214 T5636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1393214 T5636 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1393216 T5636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1393216 T5636 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1393219 T5636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1393219 T5636 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1393223 T5636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1393223 T5636 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1393227 T5636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1393227 T5636 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1393231 T5636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1393232 T5636 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1393236 T5636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1393236 T5636 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1393240 T5636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1393241 T5636 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1393244 T5636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1393245 T5636 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1393250 T5636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1393252 T5647 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fc6e55c name:ZooKeeperConnection Watcher:127.0.0.1:45432/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1393252 T5636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1393469 T5636 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1393472 T5636 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1393476 T5636 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:35447
   [junit4]   2> 1393478 T5636 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1393479 T5636 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1393479 T5636 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-002
   [junit4]   2> 1393480 T5636 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-002/'
   [junit4]   2> 1393495 T5636 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-002/solr.xml
   [junit4]   2> 1393516 T5636 oasc.CoreContainer.<init> New CoreContainer 506924388
   [junit4]   2> 1393516 T5636 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-002/]
   [junit4]   2> 1393517 T5636 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1393517 T5636 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1393517 T5636 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1393517 T5636 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1393518 T5636 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1393518 T5636 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1393518 T5636 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1393518 T5636 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1393518 T5636 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1393520 T5636 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1393520 T5636 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1393521 T5636 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1393521 T5636 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:45432/solr
   [junit4]   2> 1393521 T5636 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1393522 T5636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1393523 T5658 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62c44449 name:ZooKeeperConnection Watcher:127.0.0.1:45432 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1393524 T5636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1393526 T5636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1393529 T5660 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@437bea27 name:ZooKeeperConnection Watcher:127.0.0.1:45432/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1393529 T5636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1393532 T5636 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1393537 T5636 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1393557 T5636 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1393561 T5636 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1393571 T5636 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1393576 T5636 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1393578 T5636 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35447_z
   [junit4]   2> 1393579 T5636 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:35447_z
   [junit4]   2> 1393582 T5636 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1393585 T5636 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1393589 T5636 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:35447_z
   [junit4]   2> 1393590 T5636 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1393593 T5636 oasc.Overseer.start Overseer (id=91900487479197702-127.0.0.1:35447_z-n_0000000000) starting
   [junit4]   2> 1393596 T5636 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1393611 T5662 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1393612 T5636 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1393615 T5636 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1393618 T5636 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1393621 T5661 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1393626 T5663 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1393627 T5663 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1393628 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1393628 T5663 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1393630 T5661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1393631 T5661 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:35447/z",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:35447_z",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1393632 T5661 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 1393632 T5661 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1393635 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1393637 T5660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1394629 T5663 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1394629 T5663 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1394630 T5663 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1394631 T5663 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1394632 T5663 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1394632 T5663 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-002/collection1/'
   [junit4]   2> 1394633 T5663 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 1394634 T5663 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 1394634 T5663 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 1394666 T5663 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 1394686 T5663 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1394688 T5663 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1394693 T5663 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1394860 T5663 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1394864 T5663 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1394865 T5663 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1394879 T5663 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1394884 T5663 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1394888 T5663 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1394889 T5663 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1394890 T5663 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1394890 T5663 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1394891 T5663 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1394892 T5663 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1394892 T5663 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1394892 T5663 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1394893 T5663 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-001/control/data/
   [junit4]   2> 1394893 T5663 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3abea411
   [junit4]   2> 1394894 T5663 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-001/control/data
   [junit4]   2> 1394894 T5663 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-001/control/data/index/
   [junit4]   2> 1394895 T5663 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1394895 T5663 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-001/control/data/index
   [junit4]   2> 1394896 T5663 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=740508858, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1394897 T5663 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2beec6f5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@65686198),segFN=segments_1,generation=1}
   [junit4]   2> 1394897 T5663 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1394902 T5663 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1394902 T5663 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1394903 T5663 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1394903 T5663 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1394904 T5663 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1394904 T5663 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1394905 T5663 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1394905 T5663 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1394905 T5663 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1394906 T5663 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1394906 T5663 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1394907 T5663 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1394907 T5663 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1394908 T5663 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1394908 T5663 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1394909 T5663 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1394909 T5663 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1394913 T5663 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1394917 T5663 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1394917 T5663 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1394918 T5663 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=24.5615234375, floorSegmentMB=1.0751953125, forceMergeDeletesPctAllowed=17.926191470419468, segmentsPerTier=21.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3957502255189558
   [junit4]   2> 1394918 T5663 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2beec6f5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@65686198),segFN=segments_1,generation=1}
   [junit4]   2> 1394919 T5663 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1394919 T5663 oass.SolrIndexSearcher.<init> Opening Searcher@357a5810[collection1] main
   [junit4]   2> 1394920 T5663 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1394921 T5663 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1394921 T5663 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1394922 T5663 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1394922 T5663 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1394923 T5663 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1394924 T5663 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1394924 T5663 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1394924 T5663 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1394925 T5663 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1394927 T5664 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@357a5810[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 1394929 T5663 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1394930 T5667 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:35447/z collection:control_collection shard:shard1
   [junit4]   2> 1394930 T5636 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1394931 T5636 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1394932 T5667 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1394936 T5636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1394938 T5669 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4dfdbd84 name:ZooKeeperConnection Watcher:127.0.0.1:45432/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1394938 T5636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1394939 T5636 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1394940 T5667 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1394943 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1394944 T5667 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1394944 T5667 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C3760 name=collection1 org.apache.solr.core.SolrCore@68a77d28 url=https://127.0.0.1:35447/z/collection1 node=127.0.0.1:35447_z C3760_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:35447/z, core=collection1, node_name=127.0.0.1:35447_z}
   [junit4]   2> 1394945 T5667 C3760 P35447 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:35447/z/collection1/
   [junit4]   2> 1394944 T5636 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1394945 T5661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1394945 T5667 C3760 P35447 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1394946 T5667 C3760 P35447 oasc.SyncStrategy.syncToMe https://127.0.0.1:35447/z/collection1/ has no replicas
   [junit4]   2> 1394946 T5667 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:35447/z/collection1/ shard1
   [junit4]   2> 1394947 T5667 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1394951 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1394957 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1394962 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1395064 T5660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1395064 T5669 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1395109 T5667 oasc.ZkController.register We are https://127.0.0.1:35447/z/collection1/ and leader is https://127.0.0.1:35447/z/collection1/
   [junit4]   2> 1395109 T5667 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:35447/z
   [junit4]   2> 1395109 T5667 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1395109 T5667 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1395109 T5667 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1395111 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1395111 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1395111 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1395111 T5667 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1395112 T5661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1395113 T5661 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:35447/z",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:35447_z",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1395115 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1395153 T5636 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1395153 T5636 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1395156 T5636 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1395157 T5636 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:49373
   [junit4]   2> 1395160 T5636 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1395161 T5636 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1395161 T5636 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-003
   [junit4]   2> 1395162 T5636 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-003/'
   [junit4]   2> 1395185 T5636 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-003/solr.xml
   [junit4]   2> 1395207 T5636 oasc.CoreContainer.<init> New CoreContainer 1679240044
   [junit4]   2> 1395207 T5636 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-003/]
   [junit4]   2> 1395208 T5636 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1395208 T5636 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1395208 T5636 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1395208 T5636 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1395209 T5636 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1395209 T5636 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1395209 T5636 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1395209 T5636 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1395209 T5636 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1395212 T5636 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1395212 T5636 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1395212 T5636 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1395212 T5636 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:45432/solr
   [junit4]   2> 1395213 T5636 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1395213 T5636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1395215 T5680 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18cd7047 name:ZooKeeperConnection Watcher:127.0.0.1:45432 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1395215 T5636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1395216 T5669 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1395216 T5660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1395218 T5636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1395219 T5682 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@339f9230 name:ZooKeeperConnection Watcher:127.0.0.1:45432/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1395220 T5636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1395242 T5636 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1396247 T5636 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49373_z
   [junit4]   2> 1396249 T5636 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49373_z
   [junit4]   2> 1396264 T5683 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1396264 T5683 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1396265 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1396265 T5683 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1396266 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1396266 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1396268 T5661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1396269 T5661 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:49373/z",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49373_z",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1396270 T5661 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 1396270 T5661 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1396274 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1396377 T5669 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1396377 T5682 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1396377 T5660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1397266 T5683 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1397266 T5683 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1397267 T5683 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1397268 T5683 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1397269 T5683 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1397270 T5683 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-003/collection1/'
   [junit4]   2> 1397271 T5683 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 1397271 T5683 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 1397272 T5683 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 1397305 T5683 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 1397324 T5683 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1397326 T5683 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1397346 T5683 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1397515 T5683 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1397519 T5683 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1397520 T5683 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1397534 T5683 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1397537 T5683 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1397540 T5683 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1397541 T5683 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1397541 T5683 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1397542 T5683 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1397543 T5683 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1397543 T5683 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1397543 T5683 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1397543 T5683 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1397544 T5683 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-003/collection1/data/
   [junit4]   2> 1397544 T5683 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3abea411
   [junit4]   2> 1397545 T5683 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-003/collection1/data
   [junit4]   2> 1397545 T5683 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-003/collection1/data/index/
   [junit4]   2> 1397545 T5683 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-003/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1397545 T5683 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-003/collection1/data/index
   [junit4]   2> 1397546 T5683 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=740508858, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1397547 T5683 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1c0b157 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b3df746),segFN=segments_1,generation=1}
   [junit4]   2> 1397547 T5683 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1397552 T5683 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1397552 T5683 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1397553 T5683 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1397553 T5683 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1397553 T5683 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1397553 T5683 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1397554 T5683 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1397554 T5683 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1397554 T5683 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1397555 T5683 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1397556 T5683 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1397556 T5683 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1397556 T5683 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1397557 T5683 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1397557 T5683 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1397558 T5683 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1397559 T5683 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1397562 T5683 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1397565 T5683 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1397565 T5683 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1397566 T5683 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=24.5615234375, floorSegmentMB=1.0751953125, forceMergeDeletesPctAllowed=17.926191470419468, segmentsPerTier=21.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3957502255189558
   [junit4]   2> 1397566 T5683 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1c0b157 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b3df746),segFN=segments_1,generation=1}
   [junit4]   2> 1397567 T5683 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1397567 T5683 oass.SolrIndexSearcher.<init> Opening Searcher@49756b4b[collection1] main
   [junit4]   2> 1397567 T5683 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1397568 T5683 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1397569 T5683 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1397569 T5683 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1397569 T5683 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1397569 T5683 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1397570 T5683 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1397570 T5683 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1397570 T5683 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1397570 T5683 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1397572 T5684 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@49756b4b[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 1397573 T5683 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1397574 T5687 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:49373/z collection:collection1 shard:shard2
   [junit4]   2> 1397575 T5636 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1397575 T5636 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1397575 T5687 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1397586 T5687 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1397588 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1397588 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1397588 T5687 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1397589 T5687 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1397588 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C3761 name=collection1 org.apache.solr.core.SolrCore@40f1b92a url=https://127.0.0.1:49373/z/collection1 node=127.0.0.1:49373_z C3761_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:49373/z, core=collection1, node_name=127.0.0.1:49373_z}
   [junit4]   2> 1397589 T5687 C3761 P49373 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:49373/z/collection1/
   [junit4]   2> 1397589 T5687 C3761 P49373 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1397589 T5661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1397589 T5687 C3761 P49373 oasc.SyncStrategy.syncToMe https://127.0.0.1:49373/z/collection1/ has no replicas
   [junit4]   2> 1397590 T5687 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:49373/z/collection1/ shard2
   [junit4]   2> 1397590 T5687 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1397591 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1397595 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1397598 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1397701 T5660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1397701 T5669 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1397701 T5682 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1397745 T5687 oasc.ZkController.register We are https://127.0.0.1:49373/z/collection1/ and leader is https://127.0.0.1:49373/z/collection1/
   [junit4]   2> 1397746 T5687 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:49373/z
   [junit4]   2> 1397746 T5687 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1397746 T5687 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1397746 T5687 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1397747 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1397748 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1397748 T5687 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1397748 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1397749 T5661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1397750 T5661 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:49373/z",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49373_z",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1397752 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1397785 T5636 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 1397786 T5636 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1397791 T5636 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1397792 T5636 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:57144
   [junit4]   2> 1397794 T5636 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1397795 T5636 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1397795 T5636 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-004
   [junit4]   2> 1397795 T5636 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-004/'
   [junit4]   2> 1397813 T5636 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-004/solr.xml
   [junit4]   2> 1397833 T5636 oasc.CoreContainer.<init> New CoreContainer 631439030
   [junit4]   2> 1397833 T5636 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-004/]
   [junit4]   2> 1397834 T5636 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1397834 T5636 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1397834 T5636 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1397834 T5636 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1397835 T5636 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1397835 T5636 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1397835 T5636 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1397835 T5636 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1397836 T5636 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1397837 T5636 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1397838 T5636 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1397838 T5636 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1397838 T5636 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:45432/solr
   [junit4]   2> 1397838 T5636 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1397839 T5636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1397841 T5698 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4dbca817 name:ZooKeeperConnection Watcher:127.0.0.1:45432 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1397841 T5636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1397844 T5636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1397845 T5700 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e707058 name:ZooKeeperConnection Watcher:127.0.0.1:45432/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1397845 T5636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1397854 T5682 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1397854 T5636 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1397854 T5660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1397854 T5669 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1398865 T5636 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57144_z
   [junit4]   2> 1398867 T5636 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57144_z
   [junit4]   2> 1398883 T5701 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1398883 T5701 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1398886 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1398886 T5701 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1398886 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1398887 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1398889 T5661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1398890 T5661 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:57144/z",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:57144_z",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1398890 T5661 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 1398891 T5661 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1398894 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1398997 T5669 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1398997 T5682 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1398997 T5660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1398997 T5700 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1399886 T5701 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1399887 T5701 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1399888 T5701 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1399888 T5701 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1399889 T5701 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1399890 T5701 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-004/collection1/'
   [junit4]   2> 1399891 T5701 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 1399891 T5701 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 1399892 T5701 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 1399926 T5701 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 1399945 T5701 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1399947 T5701 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1399951 T5701 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1400123 T5701 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1400127 T5701 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1400128 T5701 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1400138 T5701 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1400141 T5701 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1400142 T5701 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1400143 T5701 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1400143 T5701 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1400144 T5701 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1400144 T5701 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1400144 T5701 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1400145 T5701 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1400145 T5701 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1400145 T5701 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-004/collection1/data/
   [junit4]   2> 1400145 T5701 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3abea411
   [junit4]   2> 1400146 T5701 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-004/collection1/data
   [junit4]   2> 1400146 T5701 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-004/collection1/data/index/
   [junit4]   2> 1400146 T5701 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-004/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1400146 T5701 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-004/collection1/data/index
   [junit4]   2> 1400147 T5701 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=740508858, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1400147 T5701 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@35b37592 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@c0e6806),segFN=segments_1,generation=1}
   [junit4]   2> 1400147 T5701 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1400151 T5701 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1400152 T5701 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1400152 T5701 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1400152 T5701 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1400152 T5701 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1400152 T5701 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1400153 T5701 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1400153 T5701 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1400153 T5701 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1400154 T5701 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1400154 T5701 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1400155 T5701 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1400155 T5701 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1400155 T5701 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1400156 T5701 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1400156 T5701 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1400157 T5701 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1400159 T5701 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1400162 T5701 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1400162 T5701 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1400162 T5701 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=24.5615234375, floorSegmentMB=1.0751953125, forceMergeDeletesPctAllowed=17.926191470419468, segmentsPerTier=21.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3957502255189558
   [junit4]   2> 1400163 T5701 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@35b37592 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@c0e6806),segFN=segments_1,generation=1}
   [junit4]   2> 1400163 T5701 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1400163 T5701 oass.SolrIndexSearcher.<init> Opening Searcher@4dbb5f2[collection1] main
   [junit4]   2> 1400164 T5701 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1400164 T5701 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1400165 T5701 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1400165 T5701 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1400165 T5701 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1400166 T5701 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1400166 T5701 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1400166 T5701 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1400167 T5701 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1400167 T5701 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1400169 T5702 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4dbb5f2[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 1400172 T5701 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1400172 T5705 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:57144/z collection:collection1 shard:shard1
   [junit4]   2> 1400173 T5636 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1400173 T5636 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1400173 T5705 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1400180 T5705 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1400182 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1400182 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1400183 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1400183 T5705 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1400183 T5705 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C3762 name=collection1 org.apache.solr.core.SolrCore@41fc5f5a url=https://127.0.0.1:57144/z/collection1 node=127.0.0.1:57144_z C3762_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:57144/z, core=collection1, node_name=127.0.0.1:57144_z}
   [junit4]   2> 1400184 T5705 C3762 P57144 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:57144/z/collection1/
   [junit4]   2> 1400184 T5705 C3762 P57144 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1400184 T5705 C3762 P57144 oasc.SyncStrategy.syncToMe https://127.0.0.1:57144/z/collection1/ has no replicas
   [junit4]   2> 1400184 T5705 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:57144/z/collection1/ shard1
   [junit4]   2> 1400184 T5661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1400184 T5705 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1400188 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1400190 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1400193 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1400295 T5669 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1400295 T5700 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1400295 T5682 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1400295 T5660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1400341 T5705 oasc.ZkController.register We are https://127.0.0.1:57144/z/collection1/ and leader is https://127.0.0.1:57144/z/collection1/
   [junit4]   2> 1400342 T5705 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:57144/z
   [junit4]   2> 1400342 T5705 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1400342 T5705 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1400342 T5705 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1400343 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1400344 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1400344 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1400344 T5705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1400345 T5661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1400345 T5661 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:57144/z",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:57144_z",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 1400347 T5660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1400381 T5636 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 1400381 T5636 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1400395 T5636 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1400396 T5636 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:43923
   [junit4]   2> 1400397 T5636 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1400398 T5636 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1400398 T5636 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-005
   [junit4]   2> 1400398 T5636 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-005/'
   [junit4]   2> 1400410 T5636 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-005/solr.xml
   [junit4]   2> 1400432 T5636 oasc.CoreContainer.<init> New CoreContainer 835826464
   [junit4]   2> 1400432 T5636 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-005/]
   [junit4]   2> 1400433 T5636 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1400433 T5636 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 

[...truncated too long message...]

(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 1478756 T5718 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1478752 T5636 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 1478753 T5738 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1478757 T5738 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-006/collection1/data [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-006/collection1/data;done=false>>]
   [junit4]   2> 1478757 T5738 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-006/collection1/data
   [junit4]   2> 1478758 T5738 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-006/collection1/data/index [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-006/collection1/data/index;done=false>>]
   [junit4]   2> 1478758 T5738 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001/tempDir-006/collection1/data/index
   [junit4]   2> 1478758 T5738 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1478759 T5738 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.MultiThreadedOCPTest-DD151805B732301F-001
   [junit4]   2> NOTE: test params are: codec=Lucene40, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=mk, timezone=Asia/Ashgabat
   [junit4]   2> NOTE: Linux 3.8.0-41-generic amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=8,threads=1,free=174588096,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [PreAnalyzedUpdateProcessorTest, TermVectorComponentDistributedTest, DirectUpdateHandlerTest, TestWordDelimiterFilterFactory, ResourceLoaderTest, CoreAdminCreateDiscoverTest, TestRandomFaceting, DeleteReplicaTest, TestSolrXmlPersistence, SuggestComponentTest, TestSolrXMLSerializer, TestQuerySenderListener, TestSchemaNameResource, TestUniqueKeyFieldResource, TestSolrQueryParserDefaultOperatorResource, TestLRUCache, TestDocumentBuilder, DistributedQueryComponentOptimizationTest, TestSolrQueryParserResource, TestLMDirichletSimilarityFactory, TestMultiCoreConfBootstrap, CollectionsAPIDistributedZkTest, NoFacetTest, CoreContainerCoreInitFailuresTest, TestArbitraryIndexDir, TestXIncludeConfig, SolrCmdDistributorTest, RequestHandlersTest, ExpressionTest, SolrPluginUtilsTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestFieldTypeCollectionResource, OverseerStatusTest, TestQueryTypes, TestQuerySenderNoQuery, BJQParserTest, HighlighterConfigTest, TestAddFieldRealTimeGet, OutputWriterTest, TestFileDictionaryLookup, SolrRequestParserTest, TestFastWriter, TestSchemaSimilarityResource, DocValuesMultiTest, TestDFRSimilarityFactory, StatsComponentTest, SpellCheckCollatorTest, TestCSVLoader, TestStressUserVersions, DisMaxRequestHandlerTest, IndexSchemaRuntimeFieldTest, ResponseHeaderTest, TestSolrJ, AbstractAnalyticsFacetTest, ShardRoutingCustomTest, NoCacheHeaderTest, SystemInfoHandlerTest, OpenExchangeRatesOrgProviderTest, PingRequestHandlerTest, CachingDirectoryFactoryTest, JSONWriterTest, TestCopyFieldCollectionResource, TestBinaryResponseWriter, DocExpirationUpdateProcessorFactoryTest, TestSweetSpotSimilarityFactory, HardAutoCommitTest, AbstractAnalyticsStatsTest, TestNoOpRegenerator, TestFiltering, QueryFacetTest, DefaultValueUpdateProcessorTest, TestAnalyzedSuggestions, TestIndexSearcher, InfoHandlerTest, TestFastOutputStream, TestCSVResponseWriter, ZkSolrClientTest, TestBinaryField, XsltUpdateRequestHandlerTest, TestSolrDeletionPolicy2, TestSolrCoreProperties, AnalysisErrorHandlingTest, HdfsBasicDistributedZkTest, FullSolrCloudDistribCmdsTest, TestStressRecovery, TestComplexPhraseQParserPlugin, TestCoreDiscovery, TestReload, TestSolrXmlPersistor, XmlUpdateRequestHandlerTest, DistanceFunctionTest, TestCursorMarkWithoutUniqueKey, TestRangeQuery, RequiredFieldsTest, AtomicUpdatesTest, DebugComponentTest, NotRequiredUniqueKeyTest, DirectSolrConnectionTest, OpenCloseCoreStressTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, LeaderElectionIntegrationTest, BasicZkTest, RecoveryZkTest, TestReplicationHandler, TestZkChroot, ZkCLITest, AutoCommitTest, TestDistributedGrouping, TestHashPartitioner, TestStressReorder, TestStressVersions, TestCoreContainer, SolrCoreTest, TestGroupingSearch, BasicFunctionalityTest, OverseerCollectionProcessorTest, HighlighterTest, SimplePostToolTest, AnalysisAfterCoreReloadTest, TestExtendedDismaxParser, SuggesterTSTTest, TestStressLucene, SpatialFilterTest, SolrCoreCheckLockOnStartupTest, SortByFunctionTest, TestRemoteStreaming, SolrInfoMBeanTest, IndexBasedSpellCheckerTest, StandardRequestHandlerTest, DirectSolrSpellCheckerTest, DocumentAnalysisRequestHandlerTest, TestOmitPositions, DocumentBuilderTest, TestIndexingPerformance, FieldAnalysisRequestHandlerTest, RegexBoostProcessorTest, TestCollationField, MBeansHandlerTest, TestPartialUpdateDeduplication, CSVRequestHandlerTest, SearchHandlerTest, CopyFieldTest, BadComponentTest, TestMergePolicyConfig, MinimalSchemaTest, TestFuzzyAnalyzedSuggestions, TestPhraseSuggestions, TestIBSimilarityFactory, LegacyHTMLStripCharFilterTest, URLClassifyProcessorTest, TestFastLRUCache, ChaosMonkeyNothingIsSafeTest, CustomCollectionTest, DeleteShardTest, MigrateRouteKeyTest, MultiThreadedOCPTest]
   [junit4] Completed on J1 in 110.21s, 1 test, 1 error <<< FAILURES!

[...truncated 229 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 410 suites, 1693 tests, 1 error, 36 ignored (15 assumptions)

Total time: 58 minutes 48 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0_60 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.8.0_05) - Build # 10511 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10511/
Java: 32bit/jdk1.8.0_05 -client -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
shard3 is not consistent.  Got 201 from https://127.0.0.1:51867/collection1lastClient and got 189 from https://127.0.0.1:34696/collection1

Stack Trace:
java.lang.AssertionError: shard3 is not consistent.  Got 201 from https://127.0.0.1:51867/collection1lastClient and got 189 from https://127.0.0.1:34696/collection1
	at __randomizedtesting.SeedInfo.seed([BEF162510CC9F334:3F17EC497B969308]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1131)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1110)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:150)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11238 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/init-core-data-001
   [junit4]   2> 928677 T3636 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 928678 T3636 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 928681 T3636 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 928682 T3636 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 928682 T3637 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 928782 T3636 oasc.ZkTestServer.run start zk server on port:49063
   [junit4]   2> 928783 T3636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 928832 T3643 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e3b8d7 name:ZooKeeperConnection Watcher:127.0.0.1:49063 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 928833 T3636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 928833 T3636 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 928837 T3636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 928838 T3645 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@386523 name:ZooKeeperConnection Watcher:127.0.0.1:49063/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 928838 T3636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 928839 T3636 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 928841 T3636 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 928843 T3636 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 928845 T3636 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 928847 T3636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 928847 T3636 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 928850 T3636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 928851 T3636 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 928853 T3636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 928853 T3636 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 928855 T3636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 928856 T3636 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 928857 T3636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 928858 T3636 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 928859 T3636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 928860 T3636 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 928862 T3636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 928862 T3636 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 928863 T3636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 928864 T3636 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 928865 T3636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 928866 T3636 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 928868 T3636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 928868 T3636 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 928870 T3636 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 928871 T3636 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 928874 T3636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 928875 T3647 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@17e1703 name:ZooKeeperConnection Watcher:127.0.0.1:49063/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 928876 T3636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 929140 T3636 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 929143 T3636 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 929146 T3636 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:51116
   [junit4]   2> 929147 T3636 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 929148 T3636 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 929148 T3636 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-002
   [junit4]   2> 929149 T3636 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-002/'
   [junit4]   2> 929162 T3636 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-002/solr.xml
   [junit4]   2> 929171 T3636 oasc.CoreContainer.<init> New CoreContainer 20737770
   [junit4]   2> 929172 T3636 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-002/]
   [junit4]   2> 929172 T3636 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 929173 T3636 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 929173 T3636 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 929173 T3636 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 929174 T3636 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 929174 T3636 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 929174 T3636 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 929175 T3636 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 929175 T3636 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 929177 T3636 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 929177 T3636 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 929178 T3636 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 929178 T3636 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49063/solr
   [junit4]   2> 929178 T3636 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 929179 T3636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 929180 T3658 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b55a1 name:ZooKeeperConnection Watcher:127.0.0.1:49063 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 929181 T3636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 929183 T3636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 929184 T3660 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1636dc8 name:ZooKeeperConnection Watcher:127.0.0.1:49063/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 929185 T3636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 929186 T3636 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 929189 T3636 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 929191 T3636 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 929194 T3636 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 929197 T3636 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 929200 T3636 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 929202 T3636 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51116_
   [junit4]   2> 929203 T3636 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51116_
   [junit4]   2> 929205 T3636 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 929207 T3636 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 929210 T3636 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:51116_
   [junit4]   2> 929211 T3636 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 929213 T3636 oasc.Overseer.start Overseer (id=91899915051991044-127.0.0.1:51116_-n_0000000000) starting
   [junit4]   2> 929215 T3636 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 929222 T3662 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 929222 T3636 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 929225 T3636 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 929226 T3636 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 929228 T3661 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 929232 T3663 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 929232 T3663 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 929233 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 929233 T3663 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 929234 T3661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 929235 T3661 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:51116",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:51116_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 929235 T3661 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 929235 T3661 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 929237 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 929243 T3660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 930234 T3663 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 930234 T3663 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 930235 T3663 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 930235 T3663 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 930236 T3663 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 930236 T3663 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-002/collection1/'
   [junit4]   2> 930237 T3663 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 930237 T3663 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 930237 T3663 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 930265 T3663 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 930276 T3663 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 930278 T3663 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 930285 T3663 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 930394 T3663 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 930395 T3663 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 930396 T3663 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 930399 T3663 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 930413 T3663 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 930414 T3663 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 930414 T3663 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/control/data/
   [junit4]   2> 930414 T3663 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@117178c
   [junit4]   2> 930415 T3663 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/control/data
   [junit4]   2> 930415 T3663 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/control/data/index/
   [junit4]   2> 930416 T3663 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 930416 T3663 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/control/data/index
   [junit4]   2> 930417 T3663 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=92.580078125, floorSegmentMB=1.53125, forceMergeDeletesPctAllowed=8.828951119659278, segmentsPerTier=21.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6057923387262849
   [junit4]   2> 930422 T3663 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 930423 T3663 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 930425 T3663 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 930426 T3663 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 930426 T3663 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 930426 T3663 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 930426 T3663 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 930427 T3663 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 930427 T3663 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 930427 T3663 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 930427 T3663 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 930428 T3663 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 930428 T3663 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 930429 T3663 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 930429 T3663 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 930429 T3663 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 930429 T3663 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 930430 T3663 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 930430 T3663 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 930432 T3663 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 930433 T3663 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 930433 T3663 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 930434 T3663 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=80.87890625, floorSegmentMB=0.728515625, forceMergeDeletesPctAllowed=7.3048084134437286, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 930435 T3663 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 930435 T3663 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 930436 T3663 oass.SolrIndexSearcher.<init> Opening Searcher@278995[collection1] main
   [junit4]   2> 930436 T3663 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 930437 T3663 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 930437 T3663 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 930438 T3663 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 930438 T3663 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 930438 T3663 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 930438 T3663 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 930439 T3663 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 930439 T3663 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 930439 T3663 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 930442 T3664 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@278995[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 930443 T3663 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 930444 T3667 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:51116 collection:control_collection shard:shard1
   [junit4]   2> 930444 T3636 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 930445 T3636 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 930445 T3667 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 930447 T3636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 930448 T3669 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@511d1c name:ZooKeeperConnection Watcher:127.0.0.1:49063/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 930449 T3636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 930450 T3636 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 930450 T3667 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 930451 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930451 T3636 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 930452 T3667 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 930452 T3667 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2693 name=collection1 org.apache.solr.core.SolrCore@1f32a76 url=https://127.0.0.1:51116/collection1 node=127.0.0.1:51116_ C2693_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:51116, core=collection1, node_name=127.0.0.1:51116_}
   [junit4]   2> 930452 T3667 C2693 P51116 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:51116/collection1/
   [junit4]   2> 930453 T3667 C2693 P51116 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 930453 T3667 C2693 P51116 oasc.SyncStrategy.syncToMe https://127.0.0.1:51116/collection1/ has no replicas
   [junit4]   2> 930453 T3667 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:51116/collection1/ shard1
   [junit4]   2> 930454 T3667 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 930454 T3661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 930456 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930458 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930460 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930562 T3660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 930562 T3669 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 930609 T3667 oasc.ZkController.register We are https://127.0.0.1:51116/collection1/ and leader is https://127.0.0.1:51116/collection1/
   [junit4]   2> 930609 T3667 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:51116
   [junit4]   2> 930609 T3667 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 930610 T3667 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 930610 T3667 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 930611 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930612 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930612 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930612 T3667 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 930613 T3661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 930613 T3661 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:51116",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:51116_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 930622 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930724 T3660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 930724 T3669 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 930740 T3636 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 930741 T3636 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 930744 T3636 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 930745 T3636 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:48159
   [junit4]   2> 930747 T3636 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 930748 T3636 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 930749 T3636 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-003
   [junit4]   2> 930749 T3636 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-003/'
   [junit4]   2> 930769 T3636 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-003/solr.xml
   [junit4]   2> 930793 T3636 oasc.CoreContainer.<init> New CoreContainer 17496519
   [junit4]   2> 930793 T3636 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-003/]
   [junit4]   2> 930794 T3636 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 930794 T3636 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 930794 T3636 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 930795 T3636 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 930795 T3636 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 930795 T3636 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 930796 T3636 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 930796 T3636 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 930796 T3636 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 930798 T3636 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 930799 T3636 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 930799 T3636 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 930799 T3636 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49063/solr
   [junit4]   2> 930800 T3636 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 930801 T3636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 930802 T3680 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1dea563 name:ZooKeeperConnection Watcher:127.0.0.1:49063 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 930802 T3636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 930804 T3636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 930805 T3682 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1454da name:ZooKeeperConnection Watcher:127.0.0.1:49063/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 930806 T3636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 930812 T3636 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 931814 T3636 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48159_
   [junit4]   2> 931815 T3636 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48159_
   [junit4]   2> 931825 T3683 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 931825 T3683 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 931826 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 931826 T3683 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 931826 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 931827 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 931828 T3661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 931829 T3661 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:48159",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:48159_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 931829 T3661 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3]
   [junit4]   2> 931829 T3661 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 931831 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 931932 T3669 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 931932 T3682 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 931932 T3660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 932827 T3683 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 932827 T3683 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 932828 T3683 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 932828 T3683 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 932829 T3683 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 932829 T3683 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-003/collection1/'
   [junit4]   2> 932830 T3683 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 932830 T3683 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 932831 T3683 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 932853 T3683 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 932864 T3683 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 932865 T3683 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 932870 T3683 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 932976 T3683 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 932978 T3683 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 932979 T3683 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 932982 T3683 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 932996 T3683 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 932997 T3683 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 932997 T3683 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty1/
   [junit4]   2> 932997 T3683 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@117178c
   [junit4]   2> 932998 T3683 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty1
   [junit4]   2> 932998 T3683 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty1/index/
   [junit4]   2> 932999 T3683 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 932999 T3683 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty1/index
   [junit4]   2> 933000 T3683 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=92.580078125, floorSegmentMB=1.53125, forceMergeDeletesPctAllowed=8.828951119659278, segmentsPerTier=21.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6057923387262849
   [junit4]   2> 933009 T3683 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 933009 T3683 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 933013 T3683 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 933013 T3683 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 933013 T3683 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 933013 T3683 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 933014 T3683 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 933014 T3683 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 933014 T3683 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 933014 T3683 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 933015 T3683 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 933015 T3683 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 933016 T3683 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 933016 T3683 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 933016 T3683 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 933017 T3683 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 933017 T3683 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 933018 T3683 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 933018 T3683 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 933020 T3683 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 933021 T3683 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 933022 T3683 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 933022 T3683 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=80.87890625, floorSegmentMB=0.728515625, forceMergeDeletesPctAllowed=7.3048084134437286, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 933023 T3683 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 933024 T3683 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 933024 T3683 oass.SolrIndexSearcher.<init> Opening Searcher@16e6aad[collection1] main
   [junit4]   2> 933024 T3683 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 933025 T3683 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 933025 T3683 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 933025 T3683 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 933026 T3683 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 933026 T3683 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 933026 T3683 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 933027 T3683 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 933027 T3683 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 933027 T3683 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 933032 T3684 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@16e6aad[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 933035 T3683 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 933035 T3687 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:48159 collection:collection1 shard:shard2
   [junit4]   2> 933036 T3636 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 933036 T3687 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 933036 T3636 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 933042 T3687 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 933045 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933045 T3687 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 933045 T3687 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 933045 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C2694 name=collection1 org.apache.solr.core.SolrCore@83c6a9 url=https://127.0.0.1:48159/collection1 node=127.0.0.1:48159_ C2694_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:48159, core=collection1, node_name=127.0.0.1:48159_}
   [junit4]   2> 933045 T3687 C2694 P48159 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:48159/collection1/
   [junit4]   2> 933046 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933046 T3687 C2694 P48159 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 933046 T3687 C2694 P48159 oasc.SyncStrategy.syncToMe https://127.0.0.1:48159/collection1/ has no replicas
   [junit4]   2> 933047 T3687 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:48159/collection1/ shard2
   [junit4]   2> 933046 T3661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 933047 T3687 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 933049 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933051 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933053 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933155 T3669 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 933155 T3682 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 933155 T3660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 933202 T3687 oasc.ZkController.register We are https://127.0.0.1:48159/collection1/ and leader is https://127.0.0.1:48159/collection1/
   [junit4]   2> 933203 T3687 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:48159
   [junit4]   2> 933203 T3687 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 933203 T3687 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 933204 T3687 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 933205 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933205 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933205 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933205 T3687 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 933206 T3661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 933207 T3661 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:48159",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:48159_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 933209 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933316 T3669 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 933316 T3660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 933316 T3682 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 933333 T3636 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 933334 T3636 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 933336 T3636 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 933337 T3636 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:60437
   [junit4]   2> 933340 T3636 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 933340 T3636 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 933340 T3636 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-004
   [junit4]   2> 933341 T3636 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-004/'
   [junit4]   2> 933354 T3636 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-004/solr.xml
   [junit4]   2> 933364 T3636 oasc.CoreContainer.<init> New CoreContainer 26869278
   [junit4]   2> 933364 T3636 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-004/]
   [junit4]   2> 933365 T3636 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 933365 T3636 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 933365 T3636 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 933366 T3636 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 933366 T3636 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 933366 T3636 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 933367 T3636 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 933367 T3636 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 933367 T3636 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 933369 T3636 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 933370 T3636 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 933370 T3636 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 933370 T3636 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49063/solr
   [junit4]   2> 933371 T3636 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 933371 T3636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 933373 T3698 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@195608f name:ZooKeeperConnection Watcher:127.0.0.1:49063 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 933373 T3636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 933375 T3636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 933376 T3700 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@19bac39 name:ZooKeeperConnection Watcher:127.0.0.1:49063/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 933376 T3636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 933381 T3636 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 934383 T3636 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60437_
   [junit4]   2> 934385 T3636 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:60437_
   [junit4]   2> 934391 T3701 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 934392 T3701 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 934392 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 934392 T3701 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 934393 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 934393 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 934394 T3661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 934395 T3661 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:60437",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:60437_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 934395 T3661 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 934395 T3661 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 934397 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 934499 T3669 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 934499 T3660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 934499 T3700 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 934499 T3682 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 935393 T3701 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 935393 T3701 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 935394 T3701 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 935394 T3701 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 935395 T3701 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 935395 T3701 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-004/collection1/'
   [junit4]   2> 935396 T3701 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 935396 T3701 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 935396 T3701 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 935420 T3701 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 935431 T3701 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 935431 T3701 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 935437 T3701 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 935544 T3701 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 935545 T3701 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 935546 T3701 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 935549 T3701 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 935562 T3701 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 935562 T3701 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 935563 T3701 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty2/
   [junit4]   2> 935563 T3701 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@117178c
   [junit4]   2> 935563 T3701 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty2
   [junit4]   2> 935564 T3701 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty2/index/
   [junit4]   2> 935564 T3701 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 935565 T3701 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty2/index
   [junit4]   2> 935565 T3701 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=92.580078125, floorSegmentMB=1.53125, forceMergeDeletesPctAllowed=8.828951119659278, segmentsPerTier=21.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6057923387262849
   [junit4]   2> 935572 T3701 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty2/index,segFN=segments_1,generation=1}
   [junit4]   2> 935572 T3701 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 935575 T3701 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 935576 T3701 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 935576 T3701 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 935576 T3701 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 935576 T3701 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 935576 T3701 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 935577 T3701 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 935577 T3701 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 935577 T3701 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 935578 T3701 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 935579 T3701 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 935579 T3701 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 935579 T3701 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 935580 T3701 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 935580 T3701 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 935581 T3701 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 935581 T3701 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 935583 T3701 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 935584 T3701 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 935584 T3701 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 935585 T3701 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=80.87890625, floorSegmentMB=0.728515625, forceMergeDeletesPctAllowed=7.3048084134437286, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 935586 T3701 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty2/index,segFN=segments_1,generation=1}
   [junit4]   2> 935586 T3701 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 935586 T3701 oass.SolrIndexSearcher.<init> Opening Searcher@e5a1d7[collection1] main
   [junit4]   2> 935587 T3701 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 935587 T3701 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 935588 T3701 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 935588 T3701 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 935588 T3701 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 935588 T3701 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 935589 T3701 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 935589 T3701 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 935589 T3701 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 935590 T3701 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 935593 T3702 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@e5a1d7[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 935597 T3701 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 935597 T3705 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:60437 collection:collection1 shard:shard3
   [junit4]   2> 935598 T3636 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 935598 T3636 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 935598 T3705 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 935606 T3705 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 935607 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 935607 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 935608 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 935608 T3705 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 935608 T3705 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2695 name=collection1 org.apache.solr.core.SolrCore@3fee31 url=https://127.0.0.1:60437/collection1 node=127.0.0.1:60437_ C2695_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:60437, core=collection1, node_name=127.0.0.1:60437_}
   [junit4]   2> 935609 T3705 C2695 P60437 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:60437/collection1/
   [junit4]   2> 935609 T3661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 935609 T3705 C2695 P60437 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 935609 T3705 C2695 P60437 oasc.SyncStrategy.syncToMe https://127.0.0.1:60437/collection1/ has no replicas
   [junit4]   2> 935610 T3705 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:60437/collection1/ shard3
   [junit4]   2> 935610 T3705 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 935612 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 935615 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 935617 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 935719 T3669 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 935719 T3660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 935719 T3682 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 935719 T3700 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 935766 T3705 oasc.ZkController.register We are https://127.0.0.1:60437/collection1/ and leader is https://127.0.0.1:60437/collection1/
   [junit4]   2> 935767 T3705 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:60437
   [junit4]   2> 935767 T3705 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 935767 T3705 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 935767 T3705 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 935768 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 935769 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 935769 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 935769 T3705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 935770 T3661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 935771 T3661 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:60437",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:60437_",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 935775 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 935877 T3669 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 935877 T3700 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 935877 T3682 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 935877 T3660 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 935879 T3636 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 935879 T3636 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 935882 T3636 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 935883 T3636 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:36488
   [junit4]   2> 935885 T3636 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 935885 T3636 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 935885 T3636 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-005
   [junit4]   2> 935886 T3636 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-005/'
   [junit4]   2> 935899 T3636 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-005/solr.xml
   [junit4]   2> 935923 T3636 oasc.CoreContainer.<init> New CoreContainer 10259546
   [junit4]   2> 935924 T3636 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-005/]
   [junit4]   2> 935924 T3636 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 935925 T3636 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 935925 T3636 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 935925 T3636 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 935925 T3636 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 935926 T3636 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 935926 T3636 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 935926 T3636 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 935927 T3636 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 935929 T3636 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 935929 T3636 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 935929 T3636 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 935930 T3636 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49063/solr
   [junit4]   2> 935930 T3636 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 935931 T3636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 935932 T3716 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f4554 name:ZooKeeperConnection Watcher:127.0.0.1:49063 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 935933 T3636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 935934 T3636 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 935935 T3718 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c32c86 name:ZooKeeperConnection Watcher:127.0.0.1:49063/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 935936 T3636 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 935941 T3636 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 936945 T3636 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36488_
   [junit4]   2> 936947 T3636 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:36488_
   [junit4]   2> 936952 T3719 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 936952 T3719 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 936953 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 936953 T3719 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 936953 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 936954 T3660 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 936954 T3661 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 936955 T3661 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:36488",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:36488_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 936955 T3661 oas

[...truncated too long message...]

[junit4]   2> 1075306 T4238 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty12 [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty12;done=false>>]
   [junit4]   2> 1075307 T4238 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty12
   [junit4]   2> 1075307 T4238 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty12/index.20140609033752297 [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty12/index.20140609033752297;done=false>>]
   [junit4]   2> 1075307 T4238 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001/tempDir-001/jetty12/index.20140609033752297
   [junit4]   2> 1075307 T4238 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/queue/qn-
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:243)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:240)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:240)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.createData(DistributedQueue.java:346)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.offer(DistributedQueue.java:330)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:190)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 1075308 T4238 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1075308 T4238 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-BEF162510CC9F334-001
   [junit4]   2> NOTE: test params are: codec=Lucene40, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=en_CA, timezone=Africa/Brazzaville
   [junit4]   2> NOTE: Linux 3.8.0-41-generic i386/Oracle Corporation 1.8.0_05 (32-bit)/cpus=8,threads=1,free=236314768,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [UnloadDistributedZkTest, JsonLoaderTest, DocValuesTest, TestGroupingSearch, TestSort, SynonymTokenizerTest, ZkCLITest, TestSearchPerf, HttpPartitionTest, TestCoreContainer, TermsComponentTest, OverseerStatusTest, RegexBoostProcessorTest, CopyFieldTest, TestReloadAndDeleteDocs, QueryElevationComponentTest, MoreLikeThisHandlerTest, BlockCacheTest, TestSolrIndexConfig, TestStressUserVersions, TestJoin, TestRealTimeGet, CursorPagingTest, TestModifyConfFiles, IndexSchemaTest, OverseerTest, TestRemoteStreaming, RecoveryZkTest, TestDocSet, QueryFacetTest, TermVectorComponentTest, CircularListTest, TestConfig, SpellingQueryConverterTest, TestFaceting, CoreAdminHandlerTest, TestStandardQParsers, AutoCommitTest, ModifyConfFileTest, TestOverriddenPrefixQueryForCustomFieldType, SearchHandlerTest, TestCopyFieldCollectionResource, PreAnalyzedFieldTest, SuggesterTest, DirectUpdateHandlerOptimizeTest, DocumentBuilderTest, SliceStateUpdateTest, SolrCoreTest, StressHdfsTest, SyncSliceTest, UpdateParamsTest, TestCursorMarkWithoutUniqueKey, DistributedQueryComponentOptimizationTest, TestSimpleQParserPlugin, TestPostingsSolrHighlighter, TestDistributedMissingSort, TestBlendedInfixSuggestions, PolyFieldTest, HdfsUnloadDistributedZkTest, TestStressVersions, TestRandomDVFaceting, QueryResultKeyTest, TestShortCircuitedRequests, TestCollectionAPI, EchoParamsTest, TimeZoneUtilsTest, TestPseudoReturnFields, TestCollationFieldDocValues, TestFieldTypeResource, AnalysisAfterCoreReloadTest, TestValueSourceCache, SimpleFacetsTest, TestPHPSerializedResponseWriter, StatelessScriptUpdateProcessorFactoryTest, AddBlockUpdateTest, QueryParsingTest, DistributedQueryElevationComponentTest, PrimitiveFieldTypeTest, TestNonDefinedSimilarityFactory, DeleteShardTest, SuggesterFSTTest, TestNonNRTOpen, ScriptEngineTest, RAMDirectoryFactoryTest, ExpressionTest, CSVRequestHandlerTest, FastVectorHighlighterTest, FieldMutatingUpdateProcessorTest, OverseerCollectionProcessorTest, ClusterStateUpdateTest, SolrInfoMBeanTest, TestComplexPhraseQParserPlugin, AssignTest, URLClassifyProcessorTest, ZkControllerTest, DateRangeFieldTest, ClusterStateTest, DistribCursorPagingTest, TestRestManager, SchemaVersionSpecificBehaviorTest, ChaosMonkeySafeLeaderTest]
   [junit4] Completed on J0 in 146.92s, 1 test, 1 failure <<< FAILURES!

[...truncated 622 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 410 suites, 1693 tests, 1 failure, 36 ignored (15 assumptions)

Total time: 65 minutes 26 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.8.0_05 -client -XX:+UseSerialGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (64bit/jdk1.8.0_20-ea-b15) - Build # 10510 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10510/
Java: 64bit/jdk1.8.0_20-ea-b15 -XX:-UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
shard3 is not consistent.  Got 437 from http://127.0.0.1:33462/y_/aj/collection1lastClient and got 422 from http://127.0.0.1:44081/y_/aj/collection1

Stack Trace:
java.lang.AssertionError: shard3 is not consistent.  Got 437 from http://127.0.0.1:33462/y_/aj/collection1lastClient and got 422 from http://127.0.0.1:44081/y_/aj/collection1
	at __randomizedtesting.SeedInfo.seed([22ACEAC452226E54:A34A64DC257D0E68]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1131)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1110)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:150)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11201 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/init-core-data-001
   [junit4]   2> 896963 T3499 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (true)
   [junit4]   2> 896963 T3499 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /y_/aj
   [junit4]   2> 896966 T3499 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 896967 T3499 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 896967 T3500 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 897067 T3499 oasc.ZkTestServer.run start zk server on port:53210
   [junit4]   2> 897068 T3499 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 897146 T3506 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a60087f name:ZooKeeperConnection Watcher:127.0.0.1:53210 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 897146 T3499 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 897147 T3499 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 897153 T3499 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 897155 T3508 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b2588d5 name:ZooKeeperConnection Watcher:127.0.0.1:53210/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 897155 T3499 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 897156 T3499 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 897160 T3499 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 897162 T3499 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 897164 T3499 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 897165 T3499 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 897166 T3499 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 897170 T3499 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 897171 T3499 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 897174 T3499 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 897175 T3499 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 897177 T3499 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 897178 T3499 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 897181 T3499 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 897181 T3499 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 897184 T3499 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 897184 T3499 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 897187 T3499 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 897188 T3499 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 897191 T3499 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 897191 T3499 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 897194 T3499 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 897195 T3499 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 897198 T3499 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 897199 T3499 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 897200 T3499 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 897201 T3499 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 897438 T3499 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 897441 T3499 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:45091
   [junit4]   2> 897442 T3499 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 897443 T3499 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 897443 T3499 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-002
   [junit4]   2> 897444 T3499 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-002/'
   [junit4]   2> 897466 T3499 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-002/solr.xml
   [junit4]   2> 897480 T3499 oasc.CoreContainer.<init> New CoreContainer 456529681
   [junit4]   2> 897480 T3499 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-002/]
   [junit4]   2> 897481 T3499 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 897481 T3499 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 897481 T3499 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 897481 T3499 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 897481 T3499 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 897482 T3499 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 897482 T3499 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 897482 T3499 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 897482 T3499 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 897483 T3499 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 897483 T3499 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 897483 T3499 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 897484 T3499 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53210/solr
   [junit4]   2> 897484 T3499 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 897485 T3499 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 897487 T3519 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7db5c24a name:ZooKeeperConnection Watcher:127.0.0.1:53210 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 897487 T3499 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 897490 T3499 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 897491 T3521 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5436462d name:ZooKeeperConnection Watcher:127.0.0.1:53210/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 897492 T3499 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 897493 T3499 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 897495 T3499 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 897496 T3499 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 897498 T3499 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 897500 T3499 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 897502 T3499 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 897503 T3499 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45091_y_%2Faj
   [junit4]   2> 897505 T3499 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45091_y_%2Faj
   [junit4]   2> 897508 T3499 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 897510 T3499 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 897512 T3499 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:45091_y_%2Faj
   [junit4]   2> 897512 T3499 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 897513 T3499 oasc.Overseer.start Overseer (id=91899395886481411-127.0.0.1:45091_y_%2Faj-n_0000000000) starting
   [junit4]   2> 897515 T3499 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 897521 T3523 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 897521 T3499 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 897523 T3499 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 897524 T3499 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 897525 T3522 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 897526 T3524 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 897526 T3524 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 897527 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 897527 T3524 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 897527 T3522 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 897528 T3522 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45091/y_/aj",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45091_y_%2Faj",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 897528 T3522 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 897528 T3522 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 897529 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 897540 T3521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 898527 T3524 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 898528 T3524 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 898529 T3524 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 898529 T3524 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 898530 T3524 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 898531 T3524 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-002/collection1/'
   [junit4]   2> 898531 T3524 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 898532 T3524 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 898532 T3524 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 898551 T3524 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 898559 T3524 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 898560 T3524 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 898564 T3524 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 898661 T3524 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 898662 T3524 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 898663 T3524 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 898665 T3524 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 898675 T3524 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 898675 T3524 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 898682 T3524 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/control/data/
   [junit4]   2> 898682 T3524 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1b177630
   [junit4]   2> 898682 T3524 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/control/data
   [junit4]   2> 898683 T3524 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/control/data/index/
   [junit4]   2> 898683 T3524 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 898683 T3524 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/control/data/index
   [junit4]   2> 898683 T3524 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=49, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=64.849609375, floorSegmentMB=1.6953125, forceMergeDeletesPctAllowed=18.807660060495177, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 898689 T3524 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 898689 T3524 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 898691 T3524 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 898691 T3524 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 898692 T3524 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 898692 T3524 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 898692 T3524 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 898692 T3524 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 898692 T3524 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 898692 T3524 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 898693 T3524 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 898693 T3524 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 898693 T3524 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 898693 T3524 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 898693 T3524 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 898693 T3524 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 898694 T3524 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 898694 T3524 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 898694 T3524 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 898696 T3524 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 898698 T3524 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 898698 T3524 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 898699 T3524 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=40, maxMergedSegmentMB=37.748046875, floorSegmentMB=0.953125, forceMergeDeletesPctAllowed=29.960859252724052, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5706901075809272
   [junit4]   2> 898700 T3524 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 898701 T3524 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 898701 T3524 oass.SolrIndexSearcher.<init> Opening Searcher@6c7dba2c[collection1] main
   [junit4]   2> 898701 T3524 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 898702 T3524 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 898702 T3524 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 898703 T3524 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 898703 T3524 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 898704 T3524 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 898705 T3524 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 898705 T3524 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 898706 T3524 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 898706 T3524 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 898708 T3525 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6c7dba2c[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 898709 T3524 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 898710 T3528 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:45091/y_/aj collection:control_collection shard:shard1
   [junit4]   2> 898710 T3499 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 898711 T3499 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 898711 T3528 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 898712 T3499 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 898713 T3530 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7078b490 name:ZooKeeperConnection Watcher:127.0.0.1:53210/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 898713 T3499 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 898714 T3499 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 898715 T3499 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 898716 T3528 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 898718 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 898718 T3528 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 898718 T3528 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1673 name=collection1 org.apache.solr.core.SolrCore@4cab35e6 url=http://127.0.0.1:45091/y_/aj/collection1 node=127.0.0.1:45091_y_%2Faj C1673_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:45091/y_/aj, core=collection1, node_name=127.0.0.1:45091_y_%2Faj}
   [junit4]   2> 898718 T3528 C1673 P45091 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:45091/y_/aj/collection1/
   [junit4]   2> 898718 T3522 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 898718 T3528 C1673 P45091 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 898719 T3528 C1673 P45091 oasc.SyncStrategy.syncToMe http://127.0.0.1:45091/y_/aj/collection1/ has no replicas
   [junit4]   2> 898719 T3528 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:45091/y_/aj/collection1/ shard1
   [junit4]   2> 898719 T3528 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 898720 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 898721 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 898722 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 898826 T3530 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 898826 T3521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 898872 T3528 oasc.ZkController.register We are http://127.0.0.1:45091/y_/aj/collection1/ and leader is http://127.0.0.1:45091/y_/aj/collection1/
   [junit4]   2> 898872 T3528 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:45091/y_/aj
   [junit4]   2> 898872 T3528 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 898872 T3528 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 898872 T3528 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 898874 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 898874 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 898874 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 898874 T3528 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 898875 T3522 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 898875 T3522 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45091/y_/aj",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45091_y_%2Faj",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 898876 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 898941 T3499 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 898942 T3499 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 898944 T3499 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:41618
   [junit4]   2> 898945 T3499 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 898946 T3499 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 898946 T3499 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-003
   [junit4]   2> 898946 T3499 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-003/'
   [junit4]   2> 898960 T3499 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-003/solr.xml
   [junit4]   2> 898981 T3499 oasc.CoreContainer.<init> New CoreContainer 1560794574
   [junit4]   2> 898981 T3499 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-003/]
   [junit4]   2> 898982 T3499 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 898983 T3499 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 898983 T3499 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 898984 T3499 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 898984 T3499 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 898984 T3499 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 898985 T3499 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 898985 T3521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 898985 T3499 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 898986 T3499 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 898985 T3530 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 898987 T3499 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 898987 T3499 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 898987 T3499 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 898988 T3499 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53210/solr
   [junit4]   2> 898988 T3499 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 898989 T3499 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 898991 T3541 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1bc98b92 name:ZooKeeperConnection Watcher:127.0.0.1:53210 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 898991 T3499 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 898993 T3499 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 898995 T3543 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@428af993 name:ZooKeeperConnection Watcher:127.0.0.1:53210/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 898995 T3499 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 898999 T3499 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 900001 T3499 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41618_y_%2Faj
   [junit4]   2> 900002 T3499 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41618_y_%2Faj
   [junit4]   2> 900008 T3544 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 900009 T3544 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 900009 T3544 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 900009 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 900010 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 900010 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 900011 T3522 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 900011 T3522 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41618/y_/aj",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:41618_y_%2Faj",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 900011 T3522 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3]
   [junit4]   2> 900012 T3522 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 900013 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 900114 T3521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 900114 T3530 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 900114 T3543 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 901010 T3544 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 901010 T3544 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 901011 T3544 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 901011 T3544 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 901013 T3544 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 901013 T3544 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-003/collection1/'
   [junit4]   2> 901014 T3544 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 901015 T3544 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 901015 T3544 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 901045 T3544 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 901059 T3544 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 901060 T3544 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 901064 T3544 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 901183 T3544 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 901184 T3544 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 901185 T3544 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 901187 T3544 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 901202 T3544 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 901203 T3544 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 901203 T3544 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty1/
   [junit4]   2> 901203 T3544 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1b177630
   [junit4]   2> 901203 T3544 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty1
   [junit4]   2> 901204 T3544 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty1/index/
   [junit4]   2> 901204 T3544 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 901204 T3544 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty1/index
   [junit4]   2> 901204 T3544 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=49, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=64.849609375, floorSegmentMB=1.6953125, forceMergeDeletesPctAllowed=18.807660060495177, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 901210 T3544 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 901211 T3544 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 901216 T3544 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 901216 T3544 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 901217 T3544 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 901217 T3544 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 901217 T3544 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 901218 T3544 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 901218 T3544 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 901219 T3544 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 901219 T3544 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 901220 T3544 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 901220 T3544 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 901221 T3544 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 901221 T3544 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 901222 T3544 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 901222 T3544 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 901223 T3544 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 901223 T3544 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 901226 T3544 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 901228 T3544 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 901228 T3544 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 901229 T3544 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=40, maxMergedSegmentMB=37.748046875, floorSegmentMB=0.953125, forceMergeDeletesPctAllowed=29.960859252724052, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5706901075809272
   [junit4]   2> 901231 T3544 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 901231 T3544 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 901231 T3544 oass.SolrIndexSearcher.<init> Opening Searcher@5eb72820[collection1] main
   [junit4]   2> 901232 T3544 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 901233 T3544 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 901233 T3544 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 901234 T3544 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 901234 T3544 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 901235 T3544 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 901236 T3544 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 901236 T3544 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 901236 T3544 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 901236 T3544 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 901240 T3545 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5eb72820[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 901243 T3544 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 901244 T3548 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:41618/y_/aj collection:collection1 shard:shard2
   [junit4]   2> 901248 T3499 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 901248 T3499 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 901249 T3548 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 901255 T3548 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 901258 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 901258 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 901258 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 901258 T3548 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 901259 T3548 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1674 name=collection1 org.apache.solr.core.SolrCore@6f6545e2 url=http://127.0.0.1:41618/y_/aj/collection1 node=127.0.0.1:41618_y_%2Faj C1674_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:41618/y_/aj, core=collection1, node_name=127.0.0.1:41618_y_%2Faj}
   [junit4]   2> 901259 T3548 C1674 P41618 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:41618/y_/aj/collection1/
   [junit4]   2> 901259 T3522 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 901259 T3548 C1674 P41618 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 901259 T3548 C1674 P41618 oasc.SyncStrategy.syncToMe http://127.0.0.1:41618/y_/aj/collection1/ has no replicas
   [junit4]   2> 901259 T3548 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:41618/y_/aj/collection1/ shard2
   [junit4]   2> 901260 T3548 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 901260 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 901262 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 901263 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 901365 T3521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 901365 T3543 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 901365 T3530 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 901413 T3548 oasc.ZkController.register We are http://127.0.0.1:41618/y_/aj/collection1/ and leader is http://127.0.0.1:41618/y_/aj/collection1/
   [junit4]   2> 901413 T3548 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:41618/y_/aj
   [junit4]   2> 901413 T3548 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 901413 T3548 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 901413 T3548 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 901414 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 901414 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 901414 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 901415 T3548 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 901415 T3522 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 901416 T3522 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41618/y_/aj",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:41618_y_%2Faj",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 901417 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 901475 T3499 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 901476 T3499 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 901478 T3499 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:33462
   [junit4]   2> 901480 T3499 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 901480 T3499 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 901481 T3499 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-004
   [junit4]   2> 901481 T3499 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-004/'
   [junit4]   2> 901499 T3499 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-004/solr.xml
   [junit4]   2> 901506 T3499 oasc.CoreContainer.<init> New CoreContainer 1174544407
   [junit4]   2> 901507 T3499 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-004/]
   [junit4]   2> 901507 T3499 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 901507 T3499 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 901508 T3499 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 901508 T3499 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 901508 T3499 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 901508 T3499 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 901508 T3499 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 901509 T3499 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 901509 T3499 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 901509 T3499 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 901510 T3499 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 901510 T3499 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 901510 T3499 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53210/solr
   [junit4]   2> 901510 T3499 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 901511 T3499 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 901513 T3559 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d37982 name:ZooKeeperConnection Watcher:127.0.0.1:53210 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 901514 T3499 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 901517 T3499 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 901518 T3561 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f23b3b6 name:ZooKeeperConnection Watcher:127.0.0.1:53210/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 901519 T3499 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 901557 T3521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 901557 T3543 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 901557 T3530 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 901562 T3499 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 902564 T3499 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33462_y_%2Faj
   [junit4]   2> 902565 T3499 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33462_y_%2Faj
   [junit4]   2> 902571 T3562 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 902571 T3562 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 902572 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 902572 T3562 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 902572 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 902572 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 902573 T3522 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 902573 T3522 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33462/y_/aj",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33462_y_%2Faj",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 902573 T3522 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 902574 T3522 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 902575 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 902677 T3530 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 902677 T3561 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 902677 T3521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 902677 T3543 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 903572 T3562 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 903572 T3562 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 903573 T3562 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 903573 T3562 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 903574 T3562 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 903574 T3562 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-004/collection1/'
   [junit4]   2> 903575 T3562 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 903575 T3562 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 903575 T3562 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 903594 T3562 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 903614 T3562 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 903615 T3562 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 903619 T3562 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 903708 T3562 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 903709 T3562 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 903710 T3562 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 903712 T3562 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 903722 T3562 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 903722 T3562 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 903723 T3562 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty2/
   [junit4]   2> 903723 T3562 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1b177630
   [junit4]   2> 903723 T3562 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty2
   [junit4]   2> 903723 T3562 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty2/index/
   [junit4]   2> 903724 T3562 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 903724 T3562 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty2/index
   [junit4]   2> 903724 T3562 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=49, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=64.849609375, floorSegmentMB=1.6953125, forceMergeDeletesPctAllowed=18.807660060495177, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 903730 T3562 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty2/index,segFN=segments_1,generation=1}
   [junit4]   2> 903730 T3562 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 903733 T3562 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 903733 T3562 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 903733 T3562 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 903733 T3562 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 903733 T3562 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 903733 T3562 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 903734 T3562 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 903734 T3562 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 903734 T3562 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 903734 T3562 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 903735 T3562 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 903735 T3562 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 903735 T3562 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 903735 T3562 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 903735 T3562 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 903736 T3562 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 903736 T3562 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 903737 T3562 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 903738 T3562 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 903738 T3562 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 903739 T3562 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=40, maxMergedSegmentMB=37.748046875, floorSegmentMB=0.953125, forceMergeDeletesPctAllowed=29.960859252724052, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5706901075809272
   [junit4]   2> 903740 T3562 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty2/index,segFN=segments_1,generation=1}
   [junit4]   2> 903740 T3562 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 903740 T3562 oass.SolrIndexSearcher.<init> Opening Searcher@3bec006d[collection1] main
   [junit4]   2> 903740 T3562 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 903741 T3562 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 903741 T3562 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 903741 T3562 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 903741 T3562 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 903741 T3562 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 903742 T3562 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 903742 T3562 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 903742 T3562 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 903742 T3562 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 903744 T3563 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3bec006d[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 903754 T3562 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 903755 T3566 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:33462/y_/aj collection:collection1 shard:shard3
   [junit4]   2> 903755 T3499 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 903755 T3566 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 903755 T3499 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 903759 T3566 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 903760 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 903760 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 903760 T3566 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 903761 T3566 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 903760 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C1675 name=collection1 org.apache.solr.core.SolrCore@14543171 url=http://127.0.0.1:33462/y_/aj/collection1 node=127.0.0.1:33462_y_%2Faj C1675_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:33462/y_/aj, core=collection1, node_name=127.0.0.1:33462_y_%2Faj}
   [junit4]   2> 903761 T3566 C1675 P33462 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:33462/y_/aj/collection1/
   [junit4]   2> 903761 T3522 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 903761 T3566 C1675 P33462 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 903761 T3566 C1675 P33462 oasc.SyncStrategy.syncToMe http://127.0.0.1:33462/y_/aj/collection1/ has no replicas
   [junit4]   2> 903762 T3566 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:33462/y_/aj/collection1/ shard3
   [junit4]   2> 903762 T3566 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 903763 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 903764 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 903766 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 903867 T3521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 903867 T3561 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 903867 T3543 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 903867 T3530 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 903915 T3566 oasc.ZkController.register We are http://127.0.0.1:33462/y_/aj/collection1/ and leader is http://127.0.0.1:33462/y_/aj/collection1/
   [junit4]   2> 903915 T3566 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:33462/y_/aj
   [junit4]   2> 903915 T3566 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 903915 T3566 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 903915 T3566 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 903916 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 903917 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 903917 T3566 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 903917 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 903918 T3522 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 903918 T3522 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33462/y_/aj",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33462_y_%2Faj",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 903923 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 903978 T3499 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 903979 T3499 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 903980 T3499 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:60630
   [junit4]   2> 903981 T3499 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 903981 T3499 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 903982 T3499 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-005
   [junit4]   2> 903982 T3499 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-005/'
   [junit4]   2> 903991 T3499 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-005/solr.xml
   [junit4]   2> 903998 T3499 oasc.CoreContainer.<init> New CoreContainer 109307600
   [junit4]   2> 903999 T3499 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-005/]
   [junit4]   2> 903999 T3499 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 903999 T3499 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 903999 T3499 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 904000 T3499 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 904000 T3499 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 904000 T3499 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 904000 T3499 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 904000 T3499 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 904001 T3499 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 904001 T3499 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 904001 T3499 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 904002 T3499 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 904002 T3499 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53210/solr
   [junit4]   2> 904002 T3499 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 904003 T3499 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 904008 T3577 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c50b5d7 name:ZooKeeperConnection Watcher:127.0.0.1:53210 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 904008 T3499 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 904010 T3499 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 904011 T3579 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b098abf name:ZooKeeperConnection Watcher:127.0.0.1:53210/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 904011 T3499 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 904019 T3499 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 904024 T3521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 904024 T3543 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 904024 T3530 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 904024 T3561 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 904024 T3579 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 905022 T3499 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60630_y_%2Faj
   [junit4]   2> 905023 T3499 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:60630_y_%2Faj
   [junit4]   2> 905029 T3580 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 905029 T3580 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 905030 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 905031 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 905031 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 905030 T3580 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 905032 T3522 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 905033 T3522 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60630/y_/aj",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:60630_y_%2Faj",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 905033 T3522 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 905034 T3522 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 905037 T3521 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type Nod

[...truncated too long message...]

tion1lastClient and got 422 from http://127.0.0.1:44081/y_/aj/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([22ACEAC452226E54:A34A64DC257D0E68]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1131)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1110)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:150)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1040222 T3499 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 1040991 T4784 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@dffe29a
   [junit4]   2> 1041003 T4784 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=1,cumulative_deletesById=3,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=12646,transaction_logs_total_number=3}
   [junit4]   2> 1041003 T4784 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1041003 T4784 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1041003 T4784 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1041006 T4784 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1041007 T4784 oasc.CachingDirectoryFactory.close Closing StandardDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1041007 T4784 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty10 [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty10;done=false>>]
   [junit4]   2> 1041007 T4784 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty10
   [junit4]   2> 1041008 T4784 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty10/index.20140608212708547 [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty10/index.20140608212708547;done=false>>]
   [junit4]   2> 1041008 T4784 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001/tempDir-001/jetty10/index.20140608212708547
   [junit4]   2> 1041008 T4784 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/queue/qn-
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:243)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:240)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:240)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.createData(DistributedQueue.java:346)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.offer(DistributedQueue.java:330)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:190)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 1041009 T4784 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-22ACEAC452226E54-001
   [junit4]   2> NOTE: test params are: codec=Lucene40, sim=DefaultSimilarity, locale=mt, timezone=SystemV/AST4ADT
   [junit4]   2> NOTE: Linux 3.8.0-41-generic amd64/Oracle Corporation 1.8.0_20-ea (64-bit)/cpus=8,threads=1,free=229307912,total=479436800
   [junit4]   2> NOTE: All tests run in this JVM: [TestSweetSpotSimilarityFactory, TestLazyCores, TestAddFieldRealTimeGet, TestLRUCache, TestDistributedSearch, TestRecovery, TestSolrDeletionPolicy2, RequestHandlersTest, SSLMigrationTest, TestCloudInspectUtil, ZkNodePropsTest, TermVectorComponentDistributedTest, HdfsCollectionsAPIDistributedZkTest, HdfsRecoveryZkTest, TestMaxScoreQueryParser, HighlighterConfigTest, MinimalSchemaTest, DOMUtilTest, ConvertedLegacyTest, TriLevelCompositeIdRoutingTest, RemoteQueryErrorTest, TestFieldCollectionResource, TestInfoStreamLogging, BasicZkTest, TestFieldTypeCollectionResource, BlockDirectoryTest, TestDFRSimilarityFactory, DirectUpdateHandlerTest, TestDefaultSearchFieldResource, TestWordDelimiterFilterFactory, FullSolrCloudDistribCmdsTest, TestReRankQParserPlugin, TestSolrQueryParserDefaultOperatorResource, ResponseHeaderTest, TestDynamicFieldCollectionResource, TestDocumentBuilder, TestRTGBase, DistanceFunctionTest, TestIBSimilarityFactory, NumericFieldsTest, CachingDirectoryFactoryTest, UniqFieldsUpdateProcessorFactoryTest, TestStressLucene, TestCollationField, TestOmitPositions, StatsComponentTest, CurrencyFieldOpenExchangeTest, LegacyHTMLStripCharFilterTest, TestManagedResource, ParsingFieldUpdateProcessorsTest, TestManagedStopFilterFactory, ConnectionManagerTest, CoreContainerCoreInitFailuresTest, BadCopyFieldTest, DateMathParserTest, AlternateDirectoryTest, SignatureUpdateProcessorFactoryTest, DistribDocExpirationUpdateProcessorTest, TestZkChroot, AnalysisErrorHandlingTest, OpenCloseCoreStressTest, CoreMergeIndexesAdminHandlerTest, SolrXmlInZkTest, BasicDistributedZk2Test, TestManagedSynonymFilterFactory, TestFreeTextSuggestions, DocumentAnalysisRequestHandlerTest, SystemInfoHandlerTest, SolrCmdDistributorTest, TestRecoveryHdfs, TestStressReorder, ShardRoutingCustomTest, TestRangeQuery, SuggesterWFSTTest, TestCollapseQParserPlugin, FieldFacetTest, SpellCheckCollatorTest, FieldFacetExtrasTest, TestReplicationHandlerBackup, TestDocBasedVersionConstraints, InfoHandlerTest, MultiThreadedOCPTest, TestBinaryField, OutputWriterTest, AbstractAnalyticsStatsTest, TestComponentsName, SimplePostToolTest, FieldAnalysisRequestHandlerTest, PingRequestHandlerTest, TestSolrJ, ChaosMonkeySafeLeaderTest]
   [junit4] Completed on J1 in 144.28s, 1 test, 1 failure <<< FAILURES!

[...truncated 663 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 410 suites, 1693 tests, 1 failure, 36 ignored (15 assumptions)

Total time: 62 minutes 41 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.8.0_20-ea-b15 -XX:-UseCompressedOops -XX:+UseSerialGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (64bit/jdk1.8.0_20-ea-b15) - Build # 10509 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10509/
Java: 64bit/jdk1.8.0_20-ea-b15 -XX:-UseCompressedOops -XX:+UseG1GC

2 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
shard2 is not consistent.  Got 236 from https://127.0.0.1:50691/cdh/gy/collection1lastClient and got 215 from https://127.0.0.1:52172/cdh/gy/collection1

Stack Trace:
java.lang.AssertionError: shard2 is not consistent.  Got 236 from https://127.0.0.1:50691/cdh/gy/collection1lastClient and got 215 from https://127.0.0.1:52172/cdh/gy/collection1
	at __randomizedtesting.SeedInfo.seed([D7D1E61AF425929C:56376802837AF2A0]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1131)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1110)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:150)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
createcollection the collection error [Watcher fired on path: null state: SyncConnected type None]

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: createcollection the collection error [Watcher fired on path: null state: SyncConnected type None]
	at __randomizedtesting.SeedInfo.seed([D7D1E61AF425929C:56376802837AF2A0]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:554)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:566)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:205)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11070 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/init-core-data-001
   [junit4]   2> 425904 T1794 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 425904 T1794 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /cdh/gy
   [junit4]   2> 425907 T1794 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 425907 T1794 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 425908 T1795 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 426008 T1794 oasc.ZkTestServer.run start zk server on port:37566
   [junit4]   2> 426010 T1794 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 426070 T1801 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4660e1e5 name:ZooKeeperConnection Watcher:127.0.0.1:37566 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 426071 T1794 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 426071 T1794 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 426077 T1794 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 426079 T1803 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e7d7e8b name:ZooKeeperConnection Watcher:127.0.0.1:37566/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 426080 T1794 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 426080 T1794 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 426085 T1794 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 426088 T1794 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 426091 T1794 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 426095 T1794 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 426096 T1794 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 426101 T1794 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 426102 T1794 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 426106 T1794 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 426107 T1794 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 426111 T1794 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 426112 T1794 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 426115 T1794 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 426116 T1794 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 426118 T1794 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 426119 T1794 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 426123 T1794 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 426124 T1794 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 426127 T1794 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 426128 T1794 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 426131 T1794 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 426132 T1794 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 426135 T1794 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 426136 T1794 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 426140 T1794 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 426140 T1794 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 426146 T1794 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 426148 T1805 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f84b3e6 name:ZooKeeperConnection Watcher:127.0.0.1:37566/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 426148 T1794 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 426372 T1794 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 426375 T1794 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 426378 T1794 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:55629
   [junit4]   2> 426379 T1794 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 426380 T1794 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 426380 T1794 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-002
   [junit4]   2> 426380 T1794 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-002/'
   [junit4]   2> 426391 T1794 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-002/solr.xml
   [junit4]   2> 426399 T1794 oasc.CoreContainer.<init> New CoreContainer 2112156018
   [junit4]   2> 426399 T1794 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-002/]
   [junit4]   2> 426400 T1794 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 426400 T1794 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 426401 T1794 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 426401 T1794 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 426401 T1794 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 426402 T1794 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 426402 T1794 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 426402 T1794 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 426402 T1794 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 426404 T1794 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 426404 T1794 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 426404 T1794 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 426405 T1794 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37566/solr
   [junit4]   2> 426405 T1794 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 426405 T1794 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 426407 T1816 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@fbe795f name:ZooKeeperConnection Watcher:127.0.0.1:37566 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 426408 T1794 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 426411 T1794 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 426413 T1818 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7dfd939a name:ZooKeeperConnection Watcher:127.0.0.1:37566/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 426414 T1794 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 426416 T1794 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 426423 T1794 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 426430 T1794 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 426434 T1794 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 426438 T1794 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 426442 T1794 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 426444 T1794 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55629_cdh%2Fgy
   [junit4]   2> 426446 T1794 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55629_cdh%2Fgy
   [junit4]   2> 426448 T1794 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 426450 T1794 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 426453 T1794 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:55629_cdh%2Fgy
   [junit4]   2> 426453 T1794 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 426455 T1794 oasc.Overseer.start Overseer (id=91898822439403524-127.0.0.1:55629_cdh%2Fgy-n_0000000000) starting
   [junit4]   2> 426457 T1794 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 426465 T1820 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 426466 T1794 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 426467 T1794 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 426468 T1794 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 426470 T1819 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 426471 T1821 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 426471 T1821 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 426472 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 426472 T1821 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 426473 T1819 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 426473 T1819 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:55629/cdh/gy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:55629_cdh%2Fgy",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 426473 T1819 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 426474 T1819 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 426475 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 426479 T1818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 427472 T1821 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 427473 T1821 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 427474 T1821 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 427474 T1821 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 427476 T1821 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 427477 T1821 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-002/collection1/'
   [junit4]   2> 427478 T1821 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 427478 T1821 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 427479 T1821 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 427508 T1821 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 427518 T1821 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 427519 T1821 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 427524 T1821 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 427616 T1821 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 427617 T1821 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 427618 T1821 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 427622 T1821 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 427645 T1821 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 427645 T1821 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 427659 T1821 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/control/data/
   [junit4]   2> 427659 T1821 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@463b09db
   [junit4]   2> 427660 T1821 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/control/data
   [junit4]   2> 427661 T1821 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/control/data/index/
   [junit4]   2> 427661 T1821 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 427662 T1821 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/control/data/index
   [junit4]   2> 427663 T1821 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=7.9267578125, floorSegmentMB=1.169921875, forceMergeDeletesPctAllowed=24.099611015611966, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4531112716772604
   [junit4]   2> 427667 T1821 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 427668 T1821 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 427672 T1821 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 427672 T1821 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 427673 T1821 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 427673 T1821 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 427673 T1821 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 427674 T1821 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 427674 T1821 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 427675 T1821 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 427675 T1821 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 427675 T1821 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 427676 T1821 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 427676 T1821 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 427677 T1821 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 427677 T1821 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 427678 T1821 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 427678 T1821 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 427679 T1821 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 427681 T1821 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 427683 T1821 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 427683 T1821 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 427684 T1821 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=34, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=91.0322265625, floorSegmentMB=1.265625, forceMergeDeletesPctAllowed=17.181881993242474, segmentsPerTier=31.0, maxCFSSegmentSizeMB=1.1953125, noCFSRatio=1.0
   [junit4]   2> 427685 T1821 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 427686 T1821 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 427686 T1821 oass.SolrIndexSearcher.<init> Opening Searcher@1254a0d[collection1] main
   [junit4]   2> 427687 T1821 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 427688 T1821 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 427688 T1821 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 427689 T1821 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 427689 T1821 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 427689 T1821 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 427690 T1821 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 427690 T1821 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 427690 T1821 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 427690 T1821 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 427691 T1822 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1254a0d[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 427692 T1821 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 427693 T1825 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:55629/cdh/gy collection:control_collection shard:shard1
   [junit4]   2> 427693 T1794 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 427693 T1794 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 427693 T1825 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 427695 T1794 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 427697 T1827 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2611609a name:ZooKeeperConnection Watcher:127.0.0.1:37566/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 427697 T1794 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 427698 T1794 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 427699 T1825 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 427702 T1794 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 427703 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 427703 T1825 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 427703 T1825 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C518 name=collection1 org.apache.solr.core.SolrCore@16176b16 url=https://127.0.0.1:55629/cdh/gy/collection1 node=127.0.0.1:55629_cdh%2Fgy C518_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:55629/cdh/gy, core=collection1, node_name=127.0.0.1:55629_cdh%2Fgy}
   [junit4]   2> 427704 T1825 C518 P55629 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:55629/cdh/gy/collection1/
   [junit4]   2> 427704 T1825 C518 P55629 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 427705 T1825 C518 P55629 oasc.SyncStrategy.syncToMe https://127.0.0.1:55629/cdh/gy/collection1/ has no replicas
   [junit4]   2> 427705 T1825 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:55629/cdh/gy/collection1/ shard1
   [junit4]   2> 427704 T1819 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 427705 T1825 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 427708 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 427711 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 427714 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 427816 T1818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 427816 T1827 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 427863 T1825 oasc.ZkController.register We are https://127.0.0.1:55629/cdh/gy/collection1/ and leader is https://127.0.0.1:55629/cdh/gy/collection1/
   [junit4]   2> 427863 T1825 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:55629/cdh/gy
   [junit4]   2> 427863 T1825 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 427863 T1825 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 427863 T1825 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 427865 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 427865 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 427865 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 427865 T1825 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 427866 T1819 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 427866 T1819 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:55629/cdh/gy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:55629_cdh%2Fgy",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 427870 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 427940 T1794 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 427941 T1794 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 427943 T1794 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 427944 T1794 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:38080
   [junit4]   2> 427946 T1794 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 427947 T1794 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 427947 T1794 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-003
   [junit4]   2> 427948 T1794 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-003/'
   [junit4]   2> 427960 T1794 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-003/solr.xml
   [junit4]   2> 427969 T1794 oasc.CoreContainer.<init> New CoreContainer 1279890953
   [junit4]   2> 427970 T1794 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-003/]
   [junit4]   2> 427970 T1794 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 427970 T1794 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 427971 T1794 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 427971 T1794 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 427971 T1794 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 427971 T1794 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 427972 T1794 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 427972 T1794 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 427972 T1827 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 427972 T1794 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 427972 T1818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 427974 T1794 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 427974 T1794 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 427975 T1794 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 427975 T1794 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37566/solr
   [junit4]   2> 427975 T1794 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 427976 T1794 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 427980 T1838 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@10d73739 name:ZooKeeperConnection Watcher:127.0.0.1:37566 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 427980 T1794 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 427983 T1794 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 427984 T1840 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f105372 name:ZooKeeperConnection Watcher:127.0.0.1:37566/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 427985 T1794 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 427992 T1794 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 428996 T1794 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38080_cdh%2Fgy
   [junit4]   2> 428999 T1794 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38080_cdh%2Fgy
   [junit4]   2> 429010 T1841 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 429011 T1841 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 429012 T1841 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 429012 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 429013 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 429013 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 429015 T1819 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 429016 T1819 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:38080/cdh/gy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38080_cdh%2Fgy",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 429016 T1819 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3]
   [junit4]   2> 429017 T1819 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 429020 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 429123 T1827 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 429123 T1840 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 429123 T1818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 430012 T1841 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 430013 T1841 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 430014 T1841 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 430014 T1841 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 430015 T1841 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 430016 T1841 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-003/collection1/'
   [junit4]   2> 430017 T1841 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 430017 T1841 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 430018 T1841 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 430045 T1841 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 430055 T1841 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 430059 T1841 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 430064 T1841 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 430159 T1841 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 430160 T1841 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 430161 T1841 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 430163 T1841 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 430174 T1841 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 430174 T1841 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 430174 T1841 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/jetty1/
   [junit4]   2> 430175 T1841 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@463b09db
   [junit4]   2> 430175 T1841 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/jetty1
   [junit4]   2> 430175 T1841 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/jetty1/index/
   [junit4]   2> 430175 T1841 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 430176 T1841 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/jetty1/index
   [junit4]   2> 430176 T1841 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=7.9267578125, floorSegmentMB=1.169921875, forceMergeDeletesPctAllowed=24.099611015611966, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4531112716772604
   [junit4]   2> 430186 T1841 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 430186 T1841 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 430189 T1841 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 430189 T1841 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 430189 T1841 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 430189 T1841 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 430189 T1841 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 430190 T1841 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 430190 T1841 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 430190 T1841 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 430190 T1841 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 430190 T1841 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 430191 T1841 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 430191 T1841 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 430191 T1841 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 430191 T1841 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 430192 T1841 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 430192 T1841 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 430192 T1841 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 430194 T1841 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 430195 T1841 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 430195 T1841 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 430195 T1841 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=34, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=91.0322265625, floorSegmentMB=1.265625, forceMergeDeletesPctAllowed=17.181881993242474, segmentsPerTier=31.0, maxCFSSegmentSizeMB=1.1953125, noCFSRatio=1.0
   [junit4]   2> 430196 T1841 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 430196 T1841 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 430196 T1841 oass.SolrIndexSearcher.<init> Opening Searcher@62206cdb[collection1] main
   [junit4]   2> 430197 T1841 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 430198 T1841 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 430198 T1841 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 430198 T1841 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 430198 T1841 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 430198 T1841 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 430199 T1841 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 430199 T1841 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 430199 T1841 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 430199 T1841 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 430202 T1842 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@62206cdb[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 430203 T1841 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 430204 T1845 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:38080/cdh/gy collection:collection1 shard:shard2
   [junit4]   2> 430204 T1794 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 430204 T1794 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 430205 T1845 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 430209 T1845 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 430211 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 430211 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 430211 T1845 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 430211 T1845 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 430211 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C519 name=collection1 org.apache.solr.core.SolrCore@335a9a20 url=https://127.0.0.1:38080/cdh/gy/collection1 node=127.0.0.1:38080_cdh%2Fgy C519_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:38080/cdh/gy, core=collection1, node_name=127.0.0.1:38080_cdh%2Fgy}
   [junit4]   2> 430211 T1845 C519 P38080 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:38080/cdh/gy/collection1/
   [junit4]   2> 430212 T1819 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 430212 T1845 C519 P38080 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 430212 T1845 C519 P38080 oasc.SyncStrategy.syncToMe https://127.0.0.1:38080/cdh/gy/collection1/ has no replicas
   [junit4]   2> 430212 T1845 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:38080/cdh/gy/collection1/ shard2
   [junit4]   2> 430212 T1845 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 430213 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 430216 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 430217 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 430319 T1827 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 430319 T1840 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 430319 T1818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 430366 T1845 oasc.ZkController.register We are https://127.0.0.1:38080/cdh/gy/collection1/ and leader is https://127.0.0.1:38080/cdh/gy/collection1/
   [junit4]   2> 430367 T1845 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:38080/cdh/gy
   [junit4]   2> 430367 T1845 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 430367 T1845 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 430367 T1845 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 430369 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 430369 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 430369 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 430370 T1845 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 430370 T1819 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 430371 T1819 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:38080/cdh/gy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38080_cdh%2Fgy",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 430375 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 430423 T1794 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 430424 T1794 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 430428 T1794 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 430430 T1794 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:37692
   [junit4]   2> 430432 T1794 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 430433 T1794 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 430434 T1794 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-004
   [junit4]   2> 430435 T1794 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-004/'
   [junit4]   2> 430451 T1794 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-004/solr.xml
   [junit4]   2> 430459 T1794 oasc.CoreContainer.<init> New CoreContainer 847873967
   [junit4]   2> 430459 T1794 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-004/]
   [junit4]   2> 430460 T1794 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 430460 T1794 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 430460 T1794 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 430460 T1794 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 430461 T1794 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 430461 T1794 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 430461 T1794 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 430461 T1794 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 430461 T1794 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 430463 T1794 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 430463 T1794 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 430464 T1794 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 430464 T1794 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37566/solr
   [junit4]   2> 430464 T1794 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 430465 T1794 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 430467 T1856 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f9bcb5f name:ZooKeeperConnection Watcher:127.0.0.1:37566 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 430467 T1794 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 430470 T1794 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 430473 T1858 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@51a7b94e name:ZooKeeperConnection Watcher:127.0.0.1:37566/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 430473 T1794 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 430477 T1827 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 430477 T1840 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 430477 T1818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 430482 T1794 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 431486 T1794 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37692_cdh%2Fgy
   [junit4]   2> 431487 T1794 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37692_cdh%2Fgy
   [junit4]   2> 431499 T1859 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 431499 T1859 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 431501 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 431501 T1859 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 431501 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 431501 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 431504 T1819 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 431504 T1819 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:37692/cdh/gy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:37692_cdh%2Fgy",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 431505 T1819 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 431505 T1819 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 431509 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 431617 T1840 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 431617 T1858 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 431617 T1818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 431617 T1827 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 432501 T1859 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 432502 T1859 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 432503 T1859 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 432503 T1859 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 432504 T1859 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 432505 T1859 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-004/collection1/'
   [junit4]   2> 432506 T1859 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 432506 T1859 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 432507 T1859 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 432540 T1859 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 432552 T1859 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 432553 T1859 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 432557 T1859 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 432648 T1859 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 432649 T1859 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 432650 T1859 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 432652 T1859 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 432664 T1859 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 432664 T1859 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 432664 T1859 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/jetty2/
   [junit4]   2> 432664 T1859 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@463b09db
   [junit4]   2> 432665 T1859 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/jetty2
   [junit4]   2> 432665 T1859 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/jetty2/index/
   [junit4]   2> 432665 T1859 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 432665 T1859 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/jetty2/index
   [junit4]   2> 432666 T1859 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=7.9267578125, floorSegmentMB=1.169921875, forceMergeDeletesPctAllowed=24.099611015611966, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4531112716772604
   [junit4]   2> 432672 T1859 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/jetty2/index,segFN=segments_1,generation=1}
   [junit4]   2> 432672 T1859 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 432675 T1859 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 432675 T1859 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 432675 T1859 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 432675 T1859 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 432676 T1859 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 432676 T1859 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 432676 T1859 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 432676 T1859 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 432676 T1859 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 432677 T1859 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 432677 T1859 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 432677 T1859 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 432678 T1859 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 432678 T1859 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 432678 T1859 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 432679 T1859 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 432679 T1859 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 432680 T1859 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 432682 T1859 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 432682 T1859 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 432682 T1859 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=34, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=91.0322265625, floorSegmentMB=1.265625, forceMergeDeletesPctAllowed=17.181881993242474, segmentsPerTier=31.0, maxCFSSegmentSizeMB=1.1953125, noCFSRatio=1.0
   [junit4]   2> 432683 T1859 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-001/jetty2/index,segFN=segments_1,generation=1}
   [junit4]   2> 432683 T1859 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 432683 T1859 oass.SolrIndexSearcher.<init> Opening Searcher@6d06d39e[collection1] main
   [junit4]   2> 432684 T1859 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 432685 T1859 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 432685 T1859 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 432685 T1859 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 432685 T1859 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 432685 T1859 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 432686 T1859 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 432686 T1859 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 432686 T1859 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 432686 T1859 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 432689 T1860 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6d06d39e[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 432691 T1859 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 432692 T1863 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:37692/cdh/gy collection:collection1 shard:shard3
   [junit4]   2> 432692 T1794 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 432692 T1794 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 432693 T1863 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 432697 T1863 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 432699 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 432699 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 432699 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 432699 T1863 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 432699 T1863 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C520 name=collection1 org.apache.solr.core.SolrCore@273751d3 url=https://127.0.0.1:37692/cdh/gy/collection1 node=127.0.0.1:37692_cdh%2Fgy C520_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:37692/cdh/gy, core=collection1, node_name=127.0.0.1:37692_cdh%2Fgy}
   [junit4]   2> 432699 T1863 C520 P37692 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:37692/cdh/gy/collection1/
   [junit4]   2> 432700 T1863 C520 P37692 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 432700 T1819 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 432700 T1863 C520 P37692 oasc.SyncStrategy.syncToMe https://127.0.0.1:37692/cdh/gy/collection1/ has no replicas
   [junit4]   2> 432700 T1863 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:37692/cdh/gy/collection1/ shard3
   [junit4]   2> 432700 T1863 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 432702 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 432704 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 432705 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 432807 T1827 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 432807 T1818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 432807 T1840 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 432807 T1858 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 432854 T1863 oasc.ZkController.register We are https://127.0.0.1:37692/cdh/gy/collection1/ and leader is https://127.0.0.1:37692/cdh/gy/collection1/
   [junit4]   2> 432855 T1863 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:37692/cdh/gy
   [junit4]   2> 432855 T1863 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 432855 T1863 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 432855 T1863 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 432859 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 432859 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 432859 T1863 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 432859 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 432860 T1819 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 432861 T1819 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:37692/cdh/gy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:37692_cdh%2Fgy",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 432862 T1818 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 432916 T1794 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 432917 T1794 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 432921 T1794 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 432923 T1794 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:36030
   [junit4]   2> 432926 T1794 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 432926 T1794 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 432927 T1794 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-D7D1E61AF425929C-001/tempDir-005
   [junit4]   2> 432927 T1794 oasc.Sol

[...truncated too long message...]

Dir-006/nodes_used_collection_shard1_replica2/data [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.CollectionsAPIDistributedZkTest-D7D1E61AF425929C-001/tempDir-006/nodes_used_collection_shard1_replica2/data;done=false>>]
   [junit4]   2> 821212 T3766 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.CollectionsAPIDistributedZkTest-D7D1E61AF425929C-001/tempDir-006/nodes_used_collection_shard1_replica2/data
   [junit4]   2> 821212 T3766 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.CollectionsAPIDistributedZkTest-D7D1E61AF425929C-001/tempDir-006/nodes_used_collection_shard1_replica2/data/index [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.CollectionsAPIDistributedZkTest-D7D1E61AF425929C-001/tempDir-006/nodes_used_collection_shard1_replica2/data/index;done=false>>]
   [junit4]   2> 821213 T3766 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.CollectionsAPIDistributedZkTest-D7D1E61AF425929C-001/tempDir-006/nodes_used_collection_shard1_replica2/data/index
   [junit4]   2> 821213 T3766 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leaders/shard1
   [junit4]   2> 821214 T3766 oasc.SolrException.log ERROR There was a problem trying to register as the leader:org.apache.solr.common.SolrException: Could not register as the leader because creating the ephemeral registration node in ZooKeeper failed
   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:135)
   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:290)
   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 		at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 		at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 	Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections
   [junit4]   2> 		at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 		at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 		at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient$3.execute(SolrZkClient.java:209)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient$3.execute(SolrZkClient.java:206)
   [junit4]   2> 		at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:206)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:417)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:386)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:373)
   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContextBase$1.execute(ElectionContext.java:127)
   [junit4]   2> 		at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:31)
   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:122)
   [junit4]   2> 		... 7 more
   [junit4]   2> 	
   [junit4]   2> 821214 T3766 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 821215 T3766 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.CollectionsAPIDistributedZkTest-D7D1E61AF425929C-001
   [junit4]   2> NOTE: test params are: codec=Lucene42, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=de_GR, timezone=America/Argentina/San_Luis
   [junit4]   2> NOTE: Linux 3.8.0-41-generic amd64/Oracle Corporation 1.8.0_20-ea (64-bit)/cpus=8,threads=1,free=180284696,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestCodecSupport, PrimUtilsTest, SpellCheckCollatorTest, TermVectorComponentDistributedTest, SolrXmlInZkTest, TestLRUCache, TestSolrXmlPersistence, JSONWriterTest, XsltUpdateRequestHandlerTest, DocumentAnalysisRequestHandlerTest, NumericFieldsTest, TestManagedSynonymFilterFactory, HighlighterConfigTest, AtomicUpdatesTest, TestCollationField, TestInfoStreamLogging, TestQuerySenderListener, FieldAnalysisRequestHandlerTest, SpellCheckComponentTest, SolrIndexConfigTest, TestCSVResponseWriter, TestFreeTextSuggestions, DirectUpdateHandlerTest, TestIBSimilarityFactory, TestRecoveryHdfs, XmlUpdateRequestHandlerTest, SolrPluginUtilsTest, SimplePostToolTest, NotRequiredUniqueKeyTest, TestOmitPositions, PluginInfoTest, SolrCoreCheckLockOnStartupTest, TestMiniSolrCloudCluster, DistanceFunctionTest, IndexSchemaRuntimeFieldTest, TestPhraseSuggestions, TestFuzzyAnalyzedSuggestions, TestQuerySenderNoQuery, TestRecovery, TestAnalyzedSuggestions, CoreMergeIndexesAdminHandlerTest, TestInitQParser, DOMUtilTest, TestFiltering, TestSchemaNameResource, TestSolrQueryParserDefaultOperatorResource, MinimalSchemaTest, TestDistributedGrouping, TestCoreDiscovery, DistribDocExpirationUpdateProcessorTest, AlternateDirectoryTest, DocValuesMultiTest, TestDynamicFieldCollectionResource, TestCloudManagedSchema, TestBinaryField, FunctionTest, TestStressReorder, HdfsBasicDistributedZk2Test, MultiTermTest, TestStressLucene, TestDocumentBuilder, TestExpandComponent, BasicDistributedZk2Test, TestWordDelimiterFilterFactory, TestFastOutputStream, AbstractAnalyticsStatsTest, TestIndexingPerformance, HdfsCollectionsAPIDistributedZkTest, TestFieldCollectionResource, SuggesterTSTTest, TestSolr4Spatial, FieldFacetExtrasTest, TestCustomSort, DeleteInactiveReplicaTest, LegacyHTMLStripCharFilterTest, TestAtomicUpdateErrorCases, TestDFRSimilarityFactory, InfoHandlerTest, TestQueryTypes, TestReplicationHandlerBackup, ConvertedLegacyTest, TestManagedResource, AnalysisErrorHandlingTest, ChaosMonkeySafeLeaderTest, TestDefaultSearchFieldResource, RequestHandlersTest, DefaultValueUpdateProcessorTest, TestSolrCoreProperties, TestLazyCores, TestClassNameShortening, OutputWriterTest, TestBinaryResponseWriter, StandardRequestHandlerTest, DateFieldTest, FullSolrCloudDistribCmdsTest, DirectSolrConnectionTest, FileUtilsTest, TestDynamicFieldResource, BasicDistributedZkTest, CollectionsAPIDistributedZkTest]
   [junit4] Completed on J1 in 125.07s, 1 test, 1 error <<< FAILURES!

[...truncated 631 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 410 suites, 1693 tests, 1 error, 1 failure, 36 ignored (15 assumptions)

Total time: 64 minutes 16 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.8.0_20-ea-b15 -XX:-UseCompressedOops -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (64bit/jdk1.7.0_60) - Build # 10508 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10508/
Java: 64bit/jdk1.7.0_60 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.SyncSliceTest.testDistribSearch

Error Message:
No live SolrServers available to handle this request

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
	at __randomizedtesting.SeedInfo.seed([BEEA2949C21CC640:3F0CA751B543A67C]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:317)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:650)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1141)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1110)
	at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:236)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11217 lines...]
   [junit4] Suite: org.apache.solr.cloud.SyncSliceTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/init-core-data-001
   [junit4]   2> 1034056 T2936 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 1034056 T2936 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1034059 T2936 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 1034059 T2936 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1034060 T2937 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1034160 T2936 oasc.ZkTestServer.run start zk server on port:34803
   [junit4]   2> 1034161 T2936 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1034229 T2943 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15d84eb2 name:ZooKeeperConnection Watcher:127.0.0.1:34803 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1034229 T2936 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1034230 T2936 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1034235 T2936 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1034236 T2945 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a490d7 name:ZooKeeperConnection Watcher:127.0.0.1:34803/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1034236 T2936 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1034236 T2936 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1034239 T2936 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1034240 T2936 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1034241 T2936 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1034243 T2936 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1034243 T2936 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1034246 T2936 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1034246 T2936 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1034251 T2936 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1034251 T2936 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1034254 T2936 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1034254 T2936 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1034256 T2936 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1034256 T2936 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1034257 T2936 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1034258 T2936 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1034259 T2936 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1034259 T2936 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1034261 T2936 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1034261 T2936 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1034263 T2936 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1034263 T2936 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1034265 T2936 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1034265 T2936 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1034267 T2936 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1034267 T2936 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1034269 T2936 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1034270 T2947 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@35d3dd41 name:ZooKeeperConnection Watcher:127.0.0.1:34803/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1034271 T2936 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1034509 T2936 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1034512 T2936 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1034514 T2936 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:51714
   [junit4]   2> 1034515 T2936 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1034515 T2936 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1034515 T2936 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-002
   [junit4]   2> 1034516 T2936 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-002/'
   [junit4]   2> 1034526 T2936 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-002/solr.xml
   [junit4]   2> 1034548 T2936 oasc.CoreContainer.<init> New CoreContainer 25098879
   [junit4]   2> 1034548 T2936 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-002/]
   [junit4]   2> 1034549 T2936 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1034549 T2936 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1034549 T2936 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1034550 T2936 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1034550 T2936 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1034550 T2936 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1034550 T2936 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1034550 T2936 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1034551 T2936 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1034553 T2936 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1034553 T2936 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1034553 T2936 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1034553 T2936 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:34803/solr
   [junit4]   2> 1034554 T2936 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1034554 T2936 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1034555 T2958 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@36a70752 name:ZooKeeperConnection Watcher:127.0.0.1:34803 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1034556 T2936 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1034560 T2936 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1034561 T2960 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3334f6f name:ZooKeeperConnection Watcher:127.0.0.1:34803/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1034561 T2936 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1034562 T2936 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1034565 T2936 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1034567 T2936 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1034568 T2936 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1034570 T2936 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1034572 T2936 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1034577 T2936 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51714_
   [junit4]   2> 1034578 T2936 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51714_
   [junit4]   2> 1034580 T2936 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1034582 T2936 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1034584 T2936 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:51714_
   [junit4]   2> 1034584 T2936 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1034586 T2936 oasc.Overseer.start Overseer (id=91897923813900292-127.0.0.1:51714_-n_0000000000) starting
   [junit4]   2> 1034588 T2936 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1034593 T2962 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1034593 T2936 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1034595 T2936 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1034596 T2936 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1034597 T2961 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1034600 T2963 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1034600 T2963 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1034601 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1034601 T2963 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1034602 T2961 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1034602 T2961 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:51714",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:51714_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1034602 T2961 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 1034602 T2961 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1034604 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1034604 T2960 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1035601 T2963 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1035601 T2963 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1035602 T2963 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1035602 T2963 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1035603 T2963 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1035603 T2963 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-002/collection1/'
   [junit4]   2> 1035603 T2963 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 1035604 T2963 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 1035604 T2963 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 1035639 T2963 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 1035663 T2963 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1035664 T2963 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1035670 T2963 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1035864 T2963 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1035869 T2963 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1035871 T2963 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1035879 T2963 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1035882 T2963 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1035886 T2963 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1035887 T2963 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1035887 T2963 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1035887 T2963 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1035888 T2963 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1035889 T2963 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1035889 T2963 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1035889 T2963 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1035889 T2963 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-001/control/data/
   [junit4]   2> 1035889 T2963 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@25e00188
   [junit4]   2> 1035890 T2963 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-001/control/data
   [junit4]   2> 1035890 T2963 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-001/control/data/index/
   [junit4]   2> 1035890 T2963 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1035891 T2963 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-001/control/data/index
   [junit4]   2> 1035891 T2963 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@7cb09e4f
   [junit4]   2> 1035891 T2963 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@59dc38c8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2f69947d),segFN=segments_1,generation=1}
   [junit4]   2> 1035892 T2963 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1035894 T2963 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1035895 T2963 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1035895 T2963 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1035895 T2963 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1035895 T2963 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1035896 T2963 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1035900 T2963 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1035900 T2963 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1035901 T2963 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1035901 T2963 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1035901 T2963 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1035902 T2963 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1035902 T2963 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1035902 T2963 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1035902 T2963 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1035903 T2963 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1035903 T2963 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1035905 T2963 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1035907 T2963 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1035908 T2963 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1035908 T2963 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1035909 T2963 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@59dc38c8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2f69947d),segFN=segments_1,generation=1}
   [junit4]   2> 1035909 T2963 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1035909 T2963 oass.SolrIndexSearcher.<init> Opening Searcher@727dab95[collection1] main
   [junit4]   2> 1035909 T2963 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1035910 T2963 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1035910 T2963 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1035911 T2963 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1035911 T2963 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1035911 T2963 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1035911 T2963 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1035911 T2963 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1035912 T2963 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1035912 T2963 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1035914 T2964 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@727dab95[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 1035915 T2963 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1035916 T2967 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:51714 collection:control_collection shard:shard1
   [junit4]   2> 1035916 T2936 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1035916 T2936 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1035916 T2967 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1035919 T2936 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1035920 T2969 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@95dc22c name:ZooKeeperConnection Watcher:127.0.0.1:34803/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1035920 T2936 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1035922 T2936 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1035922 T2967 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1035923 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1035924 T2936 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1035924 T2967 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1035924 T2967 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2137 name=collection1 org.apache.solr.core.SolrCore@5cb3cd41 url=https://127.0.0.1:51714/collection1 node=127.0.0.1:51714_ C2137_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:51714, core=collection1, node_name=127.0.0.1:51714_}
   [junit4]   2> 1035924 T2967 C2137 P51714 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:51714/collection1/
   [junit4]   2> 1035924 T2961 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1035924 T2967 C2137 P51714 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1035925 T2967 C2137 P51714 oasc.SyncStrategy.syncToMe https://127.0.0.1:51714/collection1/ has no replicas
   [junit4]   2> 1035925 T2967 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:51714/collection1/ shard1
   [junit4]   2> 1035925 T2967 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1035930 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1035932 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1035933 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1036035 T2960 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1036035 T2969 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1036083 T2967 oasc.ZkController.register We are https://127.0.0.1:51714/collection1/ and leader is https://127.0.0.1:51714/collection1/
   [junit4]   2> 1036083 T2967 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:51714
   [junit4]   2> 1036083 T2967 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1036083 T2967 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1036083 T2967 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1036085 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1036085 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1036086 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1036086 T2967 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1036087 T2961 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1036087 T2961 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:51714",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:51714_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1036089 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1036148 T2936 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1036148 T2936 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1036151 T2936 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1036152 T2936 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:43799
   [junit4]   2> 1036154 T2936 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1036154 T2936 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1036154 T2936 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-003
   [junit4]   2> 1036155 T2936 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-003/'
   [junit4]   2> 1036165 T2936 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-003/solr.xml
   [junit4]   2> 1036189 T2936 oasc.CoreContainer.<init> New CoreContainer 346685644
   [junit4]   2> 1036189 T2936 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-003/]
   [junit4]   2> 1036190 T2936 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1036190 T2936 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1036190 T2960 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1036190 T2936 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1036191 T2936 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1036191 T2936 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1036191 T2936 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1036190 T2969 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1036191 T2936 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1036192 T2936 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1036192 T2936 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1036195 T2936 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1036195 T2936 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1036195 T2936 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1036195 T2936 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:34803/solr
   [junit4]   2> 1036196 T2936 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1036196 T2936 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1036197 T2980 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@8d6b1dd name:ZooKeeperConnection Watcher:127.0.0.1:34803 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1036198 T2936 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1036200 T2936 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1036201 T2982 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3831e034 name:ZooKeeperConnection Watcher:127.0.0.1:34803/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1036201 T2936 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1036207 T2936 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1037209 T2936 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43799_
   [junit4]   2> 1037211 T2936 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43799_
   [junit4]   2> 1037227 T2983 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1037227 T2983 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1037228 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1037228 T2983 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1037229 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1037229 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1037231 T2961 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1037232 T2961 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:43799",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:43799_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1037232 T2961 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
   [junit4]   2> 1037233 T2961 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1037235 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1037338 T2960 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1037339 T2969 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1037339 T2982 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1038229 T2983 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1038229 T2983 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1038230 T2983 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1038230 T2983 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1038232 T2983 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1038232 T2983 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-003/collection1/'
   [junit4]   2> 1038233 T2983 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 1038233 T2983 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 1038234 T2983 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 1038260 T2983 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 1038283 T2983 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1038284 T2983 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1038290 T2983 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1038488 T2983 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1038493 T2983 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1038494 T2983 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1038503 T2983 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1038505 T2983 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1038507 T2983 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1038508 T2983 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1038508 T2983 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1038509 T2983 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1038509 T2983 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1038509 T2983 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1038509 T2983 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1038510 T2983 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1038510 T2983 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-001/jetty1/
   [junit4]   2> 1038510 T2983 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@25e00188
   [junit4]   2> 1038511 T2983 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-001/jetty1
   [junit4]   2> 1038511 T2983 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-001/jetty1/index/
   [junit4]   2> 1038511 T2983 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1038511 T2983 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-001/jetty1/index
   [junit4]   2> 1038511 T2983 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@10c657f6
   [junit4]   2> 1038512 T2983 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6f9c0631 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4db8e014),segFN=segments_1,generation=1}
   [junit4]   2> 1038512 T2983 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1038515 T2983 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1038516 T2983 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1038516 T2983 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1038516 T2983 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1038516 T2983 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1038516 T2983 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1038517 T2983 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1038517 T2983 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1038517 T2983 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1038517 T2983 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1038518 T2983 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1038518 T2983 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1038518 T2983 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1038518 T2983 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1038519 T2983 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1038519 T2983 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1038519 T2983 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1038521 T2983 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1038523 T2983 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1038523 T2983 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1038524 T2983 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1038524 T2983 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6f9c0631 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4db8e014),segFN=segments_1,generation=1}
   [junit4]   2> 1038524 T2983 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1038524 T2983 oass.SolrIndexSearcher.<init> Opening Searcher@bbd0ddd[collection1] main
   [junit4]   2> 1038525 T2983 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1038525 T2983 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1038526 T2983 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1038526 T2983 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1038526 T2983 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1038526 T2983 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1038527 T2983 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1038527 T2983 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1038527 T2983 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1038527 T2983 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1038529 T2984 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@bbd0ddd[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 1038531 T2983 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1038532 T2987 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:43799 collection:collection1 shard:shard1
   [junit4]   2> 1038532 T2936 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1038532 T2936 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1038532 T2987 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1038537 T2987 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1038538 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1038538 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1038538 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1038539 T2987 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1038539 T2987 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1038539 T2961 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C2138 name=collection1 org.apache.solr.core.SolrCore@7ccd994f url=https://127.0.0.1:43799/collection1 node=127.0.0.1:43799_ C2138_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:43799, core=collection1, node_name=127.0.0.1:43799_}
   [junit4]   2> 1038539 T2987 C2138 P43799 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:43799/collection1/
   [junit4]   2> 1038540 T2987 C2138 P43799 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1038540 T2987 C2138 P43799 oasc.SyncStrategy.syncToMe https://127.0.0.1:43799/collection1/ has no replicas
   [junit4]   2> 1038540 T2987 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:43799/collection1/ shard1
   [junit4]   2> 1038540 T2987 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1038541 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1038545 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1038548 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1038650 T2982 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1038650 T2960 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1038650 T2969 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1038695 T2987 oasc.ZkController.register We are https://127.0.0.1:43799/collection1/ and leader is https://127.0.0.1:43799/collection1/
   [junit4]   2> 1038695 T2987 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:43799
   [junit4]   2> 1038695 T2987 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1038695 T2987 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1038696 T2987 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1038698 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1038698 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1038698 T2987 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1038698 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1038699 T2961 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1038699 T2961 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:43799",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:43799_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1038701 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1038758 T2936 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 1038759 T2936 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1038762 T2936 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1038763 T2936 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:43504
   [junit4]   2> 1038765 T2936 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1038765 T2936 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1038765 T2936 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-004
   [junit4]   2> 1038766 T2936 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-004/'
   [junit4]   2> 1038776 T2936 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-004/solr.xml
   [junit4]   2> 1038798 T2936 oasc.CoreContainer.<init> New CoreContainer 1419178092
   [junit4]   2> 1038799 T2936 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-004/]
   [junit4]   2> 1038799 T2936 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1038800 T2936 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1038800 T2936 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1038800 T2936 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1038800 T2936 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1038801 T2936 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1038801 T2936 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1038801 T2936 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1038801 T2936 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1038802 T2960 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1038802 T2969 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1038802 T2982 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1038804 T2936 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1038804 T2936 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1038805 T2936 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1038805 T2936 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:34803/solr
   [junit4]   2> 1038805 T2936 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1038806 T2936 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1038807 T2998 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@56cd82b0 name:ZooKeeperConnection Watcher:127.0.0.1:34803 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1038808 T2936 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1038810 T2936 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1038811 T3000 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2bbd58aa name:ZooKeeperConnection Watcher:127.0.0.1:34803/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1038811 T2936 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1038824 T2936 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1039826 T2936 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43504_
   [junit4]   2> 1039827 T2936 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43504_
   [junit4]   2> 1039835 T3001 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1039835 T3001 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1039841 T3001 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1039841 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1039841 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1039841 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1039843 T2961 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1039843 T2961 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:43504",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:43504_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1039843 T2961 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=1
   [junit4]   2> 1039843 T2961 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1039845 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1039948 T2982 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1039948 T3000 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1039948 T2960 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1039948 T2969 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1040841 T3001 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1040842 T3001 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1040842 T3001 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1040843 T3001 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1040843 T3001 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1040844 T3001 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-004/collection1/'
   [junit4]   2> 1040845 T3001 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 1040845 T3001 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 1040846 T3001 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 1040876 T3001 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 1040898 T3001 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1040900 T3001 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1040905 T3001 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1041106 T3001 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1041111 T3001 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1041113 T3001 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1041123 T3001 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1041126 T3001 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1041128 T3001 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1041129 T3001 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1041129 T3001 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1041129 T3001 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1041130 T3001 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1041130 T3001 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1041130 T3001 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1041131 T3001 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1041131 T3001 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-001/jetty2/
   [junit4]   2> 1041131 T3001 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@25e00188
   [junit4]   2> 1041132 T3001 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-001/jetty2
   [junit4]   2> 1041132 T3001 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-001/jetty2/index/
   [junit4]   2> 1041132 T3001 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1041132 T3001 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-001/jetty2/index
   [junit4]   2> 1041133 T3001 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@53906b88
   [junit4]   2> 1041133 T3001 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@68a4322f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2e83d894),segFN=segments_1,generation=1}
   [junit4]   2> 1041133 T3001 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1041137 T3001 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1041137 T3001 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1041137 T3001 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1041138 T3001 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1041138 T3001 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1041138 T3001 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1041138 T3001 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1041139 T3001 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1041139 T3001 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1041139 T3001 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1041140 T3001 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1041140 T3001 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1041140 T3001 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1041141 T3001 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1041141 T3001 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1041142 T3001 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1041142 T3001 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1041145 T3001 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1041147 T3001 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1041147 T3001 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1041148 T3001 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1041148 T3001 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@68a4322f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2e83d894),segFN=segments_1,generation=1}
   [junit4]   2> 1041149 T3001 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1041149 T3001 oass.SolrIndexSearcher.<init> Opening Searcher@512fb6f[collection1] main
   [junit4]   2> 1041149 T3001 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1041150 T3001 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1041150 T3001 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1041151 T3001 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1041151 T3001 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1041151 T3001 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1041152 T3001 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1041152 T3001 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1041152 T3001 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1041152 T3001 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1041156 T3002 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@512fb6f[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 1041158 T3001 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1041158 T3005 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:43504 collection:collection1 shard:shard1
   [junit4]   2> 1041158 T2936 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1041159 T2936 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1041161 T3005 oasc.ZkController.register We are https://127.0.0.1:43504/collection1/ and leader is https://127.0.0.1:43799/collection1/
   [junit4]   2> 1041161 T3005 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:43504
   [junit4]   2> 1041161 T3005 oasc.ZkController.checkRecovery Core needs to recover:collection1
   [junit4]   2> 1041161 T3005 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
   [junit4]   2> ASYNC  NEW_CORE C2139 name=collection1 org.apache.solr.core.SolrCore@49c7948c url=https://127.0.0.1:43504/collection1 node=127.0.0.1:43504_ C2139_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:43504, core=collection1, node_name=127.0.0.1:43504_}
   [junit4]   2> 1041161 T3006 C2139 P43504 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
   [junit4]   2> 1041161 T3005 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1041162 T3006 C2139 P43504 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
   [junit4]   2> 1041162 T3006 C2139 P43504 oasc.RecoveryStrategy.doRecovery Publishing state of core collection1 as recovering, leader is https://127.0.0.1:43799/collection1/ and I am https://127.0.0.1:43504/collection1/
   [junit4]   2> 1041162 T3006 C2139 P43504 oasc.ZkController.publish publishing core=collection1 state=recovering collection=collection1
   [junit4]   2> 1041162 T3006 C2139 P43504 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1041163 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1041163 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1041164 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1041165 T3006 C2139 P43504 oasc.RecoveryStrategy.sendPrepRecoveryCmd Sending prep recovery command to https://127.0.0.1:43799; WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1%3A43504_&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true
   [junit4]   2> 1041166 T2961 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1041166 T2961 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:43504",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:43504_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 1041168 T2960 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1041220 T2978 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 1041221 T2978 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1041222 T2978 oasha.CoreAdminHandler.handleWaitForStateAction Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
   [junit4]   2> 1041222 T2978 oasha.CoreAdminHandler.handleWaitForStateAction In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:43504_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"state":"down","base_url":"https://127.0.0.1:43504","core":"collection1","node_name":"127.0.0.1:43504_"}
   [junit4]   2> 1041272 T2960 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1041273 T2982 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1041272 T2969 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1041273 T3000 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1041382 T2936 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 1041384 T2936 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1041386 T2936 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1041387 T2936 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:33440
   [junit4]   2> 1041389 T2936 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1041389 T2936 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1041389 T2936 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-005
   [junit4]   2> 1041389 T2936 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-005/'
   [junit4]   2> 1041399 T2936 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-005/solr.xml
   [junit4]   2> 1041421 T2936 oasc.CoreContainer.<init> New CoreContainer 718330128
   [junit4]   2> 1041421 T2936 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001/tempDir-005/]
   [junit4]   2> 1041422 T2936 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1041422 T2936 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1041423 T2936 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1041423 T2936 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1041423 T2936 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1041423 T2936 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1041423 T2936 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1041424 T2936 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1041424 T2936 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1041426 T2936 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1041426 T2936 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1041426 T2936 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1041426 T2936 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:34803/solr
   [junit4]   2> 1041427 T2936 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1041427 T2936 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1041429 T3018 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c183ec5 name:ZooKeeperConnection Watcher:127.0.0.1:34803 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1041429 T2936 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1041431 T2936 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1041432 T3020 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c546e36 name:ZooKeeperConnection Watcher:127.0.0.1:34803/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1041432 T2936 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1041440 T2936 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1042222 T2978 oasha.CoreAdminHandler.handleWaitForStateAction In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:43504_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"state":"recovering","base_url":"https://127.0.0.1:43504","core":"collection1","node_name":"127.0.0.1:43504_"}
   [junit4]   2> 1042223 T2978 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
   [junit4]   2> 1042223 T2978 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={checkLive=true&nodeName=127.0.0.1:43504_&wt=j

[...truncated too long message...]

solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:243)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:240)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:240)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.createData(DistributedQueue.java:346)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.offer(DistributedQueue.java:330)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:190)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 1103242 T3040 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1103242 T3040 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1103242 T3040 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1103242 T3040 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.SyncSliceTest-BEEA2949C21CC640-001
   [junit4]   2> NOTE: test params are: codec=Lucene49: {intDefault=FST41, id=PostingsFormat(name=FSTPulsing41), _version_=FST41, timestamp=PostingsFormat(name=FSTOrdPulsing41), multiDefault=PostingsFormat(name=FSTOrdPulsing41), rnd_b=PostingsFormat(name=FSTPulsing41), text=PostingsFormat(name=Lucene41VarGapFixedInterval), other_tl1=FST41, a_t=PostingsFormat(name=FSTOrdPulsing41), range_facet_l=PostingsFormat(name=FSTOrdPulsing41), a_i1=PostingsFormat(name=FSTOrdPulsing41)}, docValues:{timestamp=DocValuesFormat(name=Lucene49)}, sim=DefaultSimilarity, locale=no, timezone=Pacific/Easter
   [junit4]   2> NOTE: Linux 3.8.0-41-generic amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=8,threads=1,free=380382800,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestQueryTypes, TestXIncludeConfig, SolrCmdDistributorTest, TestManagedStopFilterFactory, TestCollationField, TestRangeQuery, TestUpdate, NotRequiredUniqueKeyTest, XsltUpdateRequestHandlerTest, BJQParserTest, TestMultiCoreConfBootstrap, TestOverriddenPrefixQueryForCustomFieldType, TestDistribDocBasedVersion, TestFuzzyAnalyzedSuggestions, TestStressUserVersions, InfoHandlerTest, DirectSolrSpellCheckerTest, SpellCheckCollatorTest, SolrRequestParserTest, TestUniqueKeyFieldResource, TestFiltering, DirectUpdateHandlerTest, ChangedSchemaMergeTest, ZkSolrClientTest, CoreMergeIndexesAdminHandlerTest, TestNoOpRegenerator, DirectSolrConnectionTest, TestSweetSpotSimilarityFactory, SuggesterTSTTest, TestOmitPositions, FieldFacetExtrasTest, HardAutoCommitTest, RequiredFieldsTest, TestDistributedGrouping, TermVectorComponentDistributedTest, TestSolrXmlPersistence, HttpPartitionTest, TestLMDirichletSimilarityFactory, TestDistributedSearch, TestQuerySenderListener, TestSchemaVersionResource, CoreContainerCoreInitFailuresTest, CoreAdminRequestStatusTest, TestPhraseSuggestions, ShardSplitTest, NoFacetTest, SimplePostToolTest, AbstractAnalyticsFacetTest, TestFieldTypeCollectionResource, RangeFacetTest, SolrTestCaseJ4Test, OpenExchangeRatesOrgProviderTest, TestFastOutputStream, TestFieldCollectionResource, TestRandomFaceting, PreAnalyzedUpdateProcessorTest, MinimalSchemaTest, NoCacheHeaderTest, TestBinaryField, TestSchemaNameResource, StandardRequestHandlerTest, DocValuesMissingTest, DocumentAnalysisRequestHandlerTest, TestSolrCoreProperties, TestCoreDiscovery, AssignTest, OutputWriterTest, DefaultValueUpdateProcessorTest, QueryFacetTest, HdfsBasicDistributedZkTest, FieldAnalysisRequestHandlerTest, SpellCheckComponentTest, TestSchemaResource, AddSchemaFieldsUpdateProcessorFactoryTest, CollectionsAPIDistributedZkTest, TestQuerySenderNoQuery, TestCSVLoader, DateMathParserTest, TestCursorMarkWithoutUniqueKey, TestCopyFieldCollectionResource, BadCopyFieldTest, TestLRUCache, JSONWriterTest, OpenCloseCoreStressTest, BasicZkTest, HdfsRecoveryZkTest, TestCloudInspectUtil, TestWordDelimiterFilterFactory, IndexBasedSpellCheckerTest, DeleteInactiveReplicaTest, BlockDirectoryTest, BadIndexSchemaTest, LegacyHTMLStripCharFilterTest, AtomicUpdatesTest, TestArbitraryIndexDir, DeleteReplicaTest, TestCodecSupport, TestSolrXmlPersistor, TestJmxIntegration, StatsComponentTest, SyncSliceTest]
   [junit4] Completed on J1 in 69.62s, 1 test, 1 error <<< FAILURES!

[...truncated 615 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 410 suites, 1693 tests, 1 error, 36 ignored (15 assumptions)

Total time: 65 minutes 13 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0_60 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.7.0_60) - Build # 10507 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10507/
Java: 32bit/jdk1.7.0_60 -client -XX:+UseSerialGC

4 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.BasicDistributedZkTest

Error Message:
ERROR: SolrZkClient opens=18 closes=17

Stack Trace:
java.lang.AssertionError: ERROR: SolrZkClient opens=18 closes=17
	at __randomizedtesting.SeedInfo.seed([B60A5BBE4920D73A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.SolrTestCaseJ4.endTrackingZkClients(SolrTestCaseJ4.java:451)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:185)
	at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:790)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.BasicDistributedZkTest

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest:     1) Thread[id=4231, name=TEST-BasicDistributedZkTest.testDistribSearch-seed#[B60A5BBE4920D73A]-EventThread, state=WAITING, group=TGRP-BasicDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    2) Thread[id=4230, name=TEST-BasicDistributedZkTest.testDistribSearch-seed#[B60A5BBE4920D73A]-SendThread(127.0.0.1:48844), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest: 
   1) Thread[id=4231, name=TEST-BasicDistributedZkTest.testDistribSearch-seed#[B60A5BBE4920D73A]-EventThread, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   2) Thread[id=4230, name=TEST-BasicDistributedZkTest.testDistribSearch-seed#[B60A5BBE4920D73A]-SendThread(127.0.0.1:48844), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
	at __randomizedtesting.SeedInfo.seed([B60A5BBE4920D73A]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.BasicDistributedZkTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=4230, name=TEST-BasicDistributedZkTest.testDistribSearch-seed#[B60A5BBE4920D73A]-SendThread(127.0.0.1:48844), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=4230, name=TEST-BasicDistributedZkTest.testDistribSearch-seed#[B60A5BBE4920D73A]-SendThread(127.0.0.1:48844), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
	at __randomizedtesting.SeedInfo.seed([B60A5BBE4920D73A]:0)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch

Error Message:
No registered leader was found after waiting for 30000ms , collection: multiunload2 slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 30000ms , collection: multiunload2 slice: shard1
	at __randomizedtesting.SeedInfo.seed([B60A5BBE4920D73A:37ECD5A63E7FB706]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:567)
	at org.apache.solr.cloud.BasicDistributedZkTest.testStopAndStartCoresInOneInstance(BasicDistributedZkTest.java:536)
	at org.apache.solr.cloud.BasicDistributedZkTest.doTest(BasicDistributedZkTest.java:368)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
	at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11227 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZkTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/init-core-data-001
   [junit4]   2> 793084 T3925 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 793088 T3925 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 793088 T3925 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 793089 T3926 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 793189 T3925 oasc.ZkTestServer.run start zk server on port:48844
   [junit4]   2> 793190 T3925 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 794538 T3929 oazsp.FileTxnLog.commit WARN fsync-ing the write ahead log in SyncThread:0 took 1347ms which will adversely effect operation latency. See the ZooKeeper troubleshooting guide
   [junit4]   2> 794540 T3932 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f5e5e8 name:ZooKeeperConnection Watcher:127.0.0.1:48844 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 794540 T3925 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 794541 T3925 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 794562 T3925 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 794581 T3934 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1fcbc1f name:ZooKeeperConnection Watcher:127.0.0.1:48844/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 794581 T3925 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 794582 T3925 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 794621 T3925 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 794641 T3925 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 794661 T3925 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 794681 T3925 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 794682 T3925 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 795044 T3925 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 795045 T3925 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 795048 T3925 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 795048 T3925 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 795050 T3925 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 795051 T3925 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 795052 T3925 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 795053 T3925 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 795054 T3925 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 795055 T3925 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 795057 T3925 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 795057 T3925 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 795060 T3925 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 795061 T3925 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 795063 T3925 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 795063 T3925 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 795065 T3925 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 795066 T3925 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 795069 T3925 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 795070 T3925 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 795311 T3925 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 795324 T3925 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:48359
   [junit4]   2> 795324 T3925 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 795325 T3925 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 795325 T3925 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-002
   [junit4]   2> 795325 T3925 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-002/'
   [junit4]   2> 795341 T3925 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-002/solr.xml
   [junit4]   2> 795373 T3925 oasc.CoreContainer.<init> New CoreContainer 31916713
   [junit4]   2> 795373 T3925 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-002/]
   [junit4]   2> 795374 T3925 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 795375 T3925 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 795375 T3925 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 795375 T3925 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 795375 T3925 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 795376 T3925 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 795376 T3925 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 795376 T3925 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 795377 T3925 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 795377 T3925 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 795378 T3925 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 795378 T3925 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 795378 T3925 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48844/solr
   [junit4]   2> 795379 T3925 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 795380 T3925 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 795381 T3945 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15fe4cb name:ZooKeeperConnection Watcher:127.0.0.1:48844 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 795381 T3925 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 795383 T3925 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 795384 T3947 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c83ac6 name:ZooKeeperConnection Watcher:127.0.0.1:48844/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 795384 T3925 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 795385 T3925 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 795388 T3925 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 795390 T3925 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 795392 T3925 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 795394 T3925 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 795397 T3925 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 795399 T3925 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48359_
   [junit4]   2> 795401 T3925 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48359_
   [junit4]   2> 795411 T3925 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 795413 T3925 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 795416 T3925 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:48359_
   [junit4]   2> 795417 T3925 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 795418 T3925 oasc.Overseer.start Overseer (id=91897420819464195-127.0.0.1:48359_-n_0000000000) starting
   [junit4]   2> 795420 T3925 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 795427 T3949 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 795428 T3925 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 795431 T3925 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 795433 T3925 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 795435 T3948 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 795438 T3950 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 795439 T3950 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 795440 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 795440 T3950 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 795441 T3948 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 795441 T3948 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48359",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:48359_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 795441 T3948 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 795442 T3948 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 795444 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 795444 T3947 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 796440 T3950 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 796441 T3950 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 796442 T3950 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 796442 T3950 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 796443 T3950 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 796443 T3950 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-002/collection1/'
   [junit4]   2> 796443 T3950 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 796444 T3950 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 796444 T3950 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 796476 T3950 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 796513 T3950 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 796514 T3950 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 796527 T3950 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 796787 T3950 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 796792 T3950 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 796794 T3950 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 796803 T3950 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 796805 T3950 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 796808 T3950 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 796809 T3950 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 796809 T3950 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 796809 T3950 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 796810 T3950 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 796810 T3950 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 796811 T3950 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 796811 T3950 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 796811 T3950 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-001/control/data/
   [junit4]   2> 796811 T3950 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ea2883
   [junit4]   2> 796812 T3950 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-001/control/data
   [junit4]   2> 796812 T3950 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-001/control/data/index/
   [junit4]   2> 796812 T3950 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 796813 T3950 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-001/control/data/index
   [junit4]   2> 796813 T3950 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=1.06640625, floorSegmentMB=2.0263671875, forceMergeDeletesPctAllowed=4.06007193541905, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3352100362648172
   [junit4]   2> 796814 T3950 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1762c7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1451c71),segFN=segments_1,generation=1}
   [junit4]   2> 796814 T3950 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 796817 T3950 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 796817 T3950 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 796817 T3950 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 796818 T3950 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 796818 T3950 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 796818 T3950 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 796818 T3950 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 796819 T3950 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 796819 T3950 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 796819 T3950 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 796820 T3950 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 796820 T3950 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 796820 T3950 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 796820 T3950 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 796821 T3950 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 796821 T3950 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 796822 T3950 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 796824 T3950 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 796826 T3950 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 796826 T3950 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 796827 T3950 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=14.36328125, floorSegmentMB=1.826171875, forceMergeDeletesPctAllowed=0.19197462011490196, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 796828 T3950 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1762c7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1451c71),segFN=segments_1,generation=1}
   [junit4]   2> 796828 T3950 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 796828 T3950 oass.SolrIndexSearcher.<init> Opening Searcher@247b3f[collection1] main
   [junit4]   2> 796828 T3950 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 796829 T3950 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 796829 T3950 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 796830 T3950 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 796830 T3950 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 796830 T3950 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 796831 T3950 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 796831 T3950 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 796831 T3950 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 796831 T3950 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 796834 T3951 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@247b3f[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 796835 T3950 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 796836 T3954 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:48359 collection:control_collection shard:shard1
   [junit4]   2> 796837 T3925 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 796837 T3925 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 796838 T3925 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 796838 T3954 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 796840 T3956 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@177229b name:ZooKeeperConnection Watcher:127.0.0.1:48844/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 796840 T3925 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 796841 T3925 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 796843 T3925 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 796844 T3954 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 796845 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 796846 T3954 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 796847 T3954 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1197 name=collection1 org.apache.solr.core.SolrCore@1a57345 url=http://127.0.0.1:48359/collection1 node=127.0.0.1:48359_ C1197_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:48359, core=collection1, node_name=127.0.0.1:48359_}
   [junit4]   2> 796847 T3954 C1197 P48359 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:48359/collection1/
   [junit4]   2> 796847 T3948 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 796847 T3954 C1197 P48359 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 796848 T3954 C1197 P48359 oasc.SyncStrategy.syncToMe http://127.0.0.1:48359/collection1/ has no replicas
   [junit4]   2> 796848 T3954 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:48359/collection1/ shard1
   [junit4]   2> 796848 T3954 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 796850 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 796853 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 796855 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 796957 T3956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 796957 T3947 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 797004 T3954 oasc.ZkController.register We are http://127.0.0.1:48359/collection1/ and leader is http://127.0.0.1:48359/collection1/
   [junit4]   2> 797004 T3954 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:48359
   [junit4]   2> 797005 T3954 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 797005 T3954 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 797005 T3954 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 797006 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 797007 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 797007 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 797007 T3954 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 797008 T3948 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 797009 T3948 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48359",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:48359_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 797010 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 797099 T3925 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 797100 T3925 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 797102 T3925 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53549
   [junit4]   2> 797103 T3925 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 797103 T3925 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 797104 T3925 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-003
   [junit4]   2> 797104 T3925 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-003/'
   [junit4]   2> 797113 T3947 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 797113 T3956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 797117 T3925 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-003/solr.xml
   [junit4]   2> 797157 T3925 oasc.CoreContainer.<init> New CoreContainer 14467823
   [junit4]   2> 797157 T3925 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-003/]
   [junit4]   2> 797158 T3925 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 797158 T3925 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 797159 T3925 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 797159 T3925 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 797159 T3925 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 797160 T3925 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 797160 T3925 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 797160 T3925 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 797160 T3925 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 797161 T3925 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 797161 T3925 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 797162 T3925 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 797162 T3925 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48844/solr
   [junit4]   2> 797162 T3925 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 797163 T3925 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 797164 T3967 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18216e6 name:ZooKeeperConnection Watcher:127.0.0.1:48844 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 797165 T3925 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 797167 T3925 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 797168 T3969 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b57e8f name:ZooKeeperConnection Watcher:127.0.0.1:48844/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 797168 T3925 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 797172 T3925 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 798175 T3925 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53549_
   [junit4]   2> 798178 T3925 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53549_
   [junit4]   2> 798191 T3970 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 798192 T3970 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 798200 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 798200 T3970 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 798200 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 798201 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 798203 T3948 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 798204 T3948 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53549",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:53549_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 798204 T3948 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 798205 T3948 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 798209 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 798311 T3969 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 798311 T3956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 798311 T3947 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 799201 T3970 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 799202 T3970 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 799203 T3970 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 799203 T3970 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 799205 T3970 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 799206 T3970 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-003/collection1/'
   [junit4]   2> 799207 T3970 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 799208 T3970 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 799208 T3970 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 799244 T3970 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 799273 T3970 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 799274 T3970 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 799282 T3970 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 799549 T3970 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 799554 T3970 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 799556 T3970 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 799565 T3970 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 799567 T3970 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 799570 T3970 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 799571 T3970 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 799572 T3970 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 799572 T3970 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 799573 T3970 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 799573 T3970 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 799573 T3970 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 799573 T3970 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 799574 T3970 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-001/jetty1/
   [junit4]   2> 799574 T3970 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ea2883
   [junit4]   2> 799574 T3970 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-001/jetty1
   [junit4]   2> 799575 T3970 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-001/jetty1/index/
   [junit4]   2> 799575 T3970 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 799575 T3970 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-001/jetty1/index
   [junit4]   2> 799576 T3970 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=1.06640625, floorSegmentMB=2.0263671875, forceMergeDeletesPctAllowed=4.06007193541905, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3352100362648172
   [junit4]   2> 799576 T3970 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@425b61 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@b66114),segFN=segments_1,generation=1}
   [junit4]   2> 799576 T3970 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 799580 T3970 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 799580 T3970 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 799581 T3970 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 799581 T3970 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 799581 T3970 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 799581 T3970 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 799582 T3970 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 799582 T3970 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 799582 T3970 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 799583 T3970 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 799584 T3970 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 799584 T3970 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 799584 T3970 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 799585 T3970 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 799585 T3970 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 799586 T3970 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 799586 T3970 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 799589 T3970 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 799592 T3970 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 799592 T3970 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 799593 T3970 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=14.36328125, floorSegmentMB=1.826171875, forceMergeDeletesPctAllowed=0.19197462011490196, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 799593 T3970 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@425b61 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@b66114),segFN=segments_1,generation=1}
   [junit4]   2> 799593 T3970 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 799594 T3970 oass.SolrIndexSearcher.<init> Opening Searcher@f5d0ed[collection1] main
   [junit4]   2> 799594 T3970 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 799595 T3970 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 799595 T3970 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 799595 T3970 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 799596 T3970 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 799596 T3970 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 799596 T3970 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 799596 T3970 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 799597 T3970 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 799597 T3970 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 799611 T3971 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@f5d0ed[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 799615 T3970 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 799616 T3974 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:53549 collection:collection1 shard:shard2
   [junit4]   2> 799616 T3925 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 799616 T3925 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 799616 T3974 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 799621 T3974 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 799622 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 799622 T3974 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 799622 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 799622 T3974 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 799623 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C1198 name=collection1 org.apache.solr.core.SolrCore@1bc7c15 url=http://127.0.0.1:53549/collection1 node=127.0.0.1:53549_ C1198_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:53549, core=collection1, node_name=127.0.0.1:53549_}
   [junit4]   2> 799623 T3974 C1198 P53549 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:53549/collection1/
   [junit4]   2> 799623 T3974 C1198 P53549 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 799623 T3974 C1198 P53549 oasc.SyncStrategy.syncToMe http://127.0.0.1:53549/collection1/ has no replicas
   [junit4]   2> 799624 T3974 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:53549/collection1/ shard2
   [junit4]   2> 799624 T3948 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 799624 T3974 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 799626 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 799627 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 799630 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 799735 T3956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 799735 T3947 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 799735 T3969 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 799778 T3974 oasc.ZkController.register We are http://127.0.0.1:53549/collection1/ and leader is http://127.0.0.1:53549/collection1/
   [junit4]   2> 799778 T3974 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:53549
   [junit4]   2> 799779 T3974 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 799779 T3974 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 799779 T3974 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 799780 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 799780 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 799781 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 799781 T3974 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 799782 T3948 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 799782 T3948 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53549",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:53549_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 799784 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 799886 T3947 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 799886 T3925 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 799886 T3969 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 799886 T3956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 799887 T3925 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 799889 T3925 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:42026
   [junit4]   2> 799890 T3925 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 799890 T3925 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 799891 T3925 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-004
   [junit4]   2> 799891 T3925 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-004/'
   [junit4]   2> 799904 T3925 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-004/solr.xml
   [junit4]   2> 799932 T3925 oasc.CoreContainer.<init> New CoreContainer 9404673
   [junit4]   2> 799932 T3925 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-004/]
   [junit4]   2> 799933 T3925 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 799934 T3925 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 799934 T3925 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 799934 T3925 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 799935 T3925 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 799935 T3925 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 799935 T3925 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 799935 T3925 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 799936 T3925 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 799936 T3925 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 799937 T3925 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 799937 T3925 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 799937 T3925 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48844/solr
   [junit4]   2> 799938 T3925 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 799938 T3925 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 799940 T3985 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15e8cd7 name:ZooKeeperConnection Watcher:127.0.0.1:48844 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 799940 T3925 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 799942 T3925 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 799943 T3987 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f294c2 name:ZooKeeperConnection Watcher:127.0.0.1:48844/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 799943 T3925 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 799948 T3925 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 800950 T3925 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42026_
   [junit4]   2> 800952 T3925 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42026_
   [junit4]   2> 800966 T3988 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 800966 T3988 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 800973 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 800973 T3988 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 800974 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 800975 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 800976 T3948 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 800977 T3948 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42026",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42026_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 800977 T3948 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 800978 T3948 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 800982 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 801085 T3969 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 801085 T3956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 801085 T3987 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 801085 T3947 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 801974 T3988 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 801975 T3988 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 801976 T3988 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 801976 T3988 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 801977 T3988 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 801978 T3988 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-004/collection1/'
   [junit4]   2> 801979 T3988 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 801979 T3988 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 801979 T3988 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 802034 T3988 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 802065 T3988 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 802067 T3988 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 802079 T3988 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 802341 T3988 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 802346 T3988 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 802348 T3988 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 802356 T3988 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 802359 T3988 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 802362 T3988 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 802363 T3988 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 802363 T3988 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 802363 T3988 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 802364 T3988 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 802364 T3988 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 802364 T3988 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 802365 T3988 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 802365 T3988 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-001/jetty2/
   [junit4]   2> 802365 T3988 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ea2883
   [junit4]   2> 802366 T3988 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-001/jetty2
   [junit4]   2> 802366 T3988 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-001/jetty2/index/
   [junit4]   2> 802366 T3988 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 802367 T3988 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.BasicDistributedZkTest-B60A5BBE4920D73A-001/tempDir-001/jetty2/index
   [junit4]   2> 802367 T3988 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=1.06640625, floorSegmentMB=2.0263671875, forceMergeDeletesPctAllowed=4.06007193541905, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3352100362648172
   [junit4]   2> 802368 T3988 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@669451 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@24c01e),segFN=segments_1,generation=1}
   [junit4]   2> 802368 T3988 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 802371 T3988 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 802371 T3988 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 802372 T3988 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 802372 T3988 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 802372 T3988 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 802372 T3988 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 802373 T3988 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 802373 T3988 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 802373 T3988 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 802374 T3988 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 802374 T3988 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 802374 T3988 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 802375 T3988 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 802375 T3988 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 802375 T3988 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 802376 T3988 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 802377 T3988 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 802379 T3988 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 802381 T3988 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 802381 T3988 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 802382 T3988 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=14.36328125, floorSegmentMB=1.826171875, forceMergeDeletesPctAllowed=0.19197462011490196, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 802383 T3988 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@669451 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@24c01e),segFN=segments_1,generation=1}
   [junit4]   2> 802383 T3988 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 802383 T3988 oass.SolrIndexSearcher.<init> Opening Searcher@19cbb6f[collection1] main
   [junit4]   2> 802383 T3988 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 802384 T3988 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 802384 T3988 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 802385 T3988 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 802385 T3988 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 802385 T3988 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 802385 T3988 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 802386 T3988 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 802386 T3988 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 802386 T3988 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 802390 T3989 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@19cbb6f[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 802394 T3988 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 802395 T3992 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:42026 collection:collection1 shard:shard1
   [junit4]   2> 802396 T3925 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 802396 T3925 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 802402 T3992 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 802406 T3992 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 802407 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 802408 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 802408 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 802408 T3992 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 802408 T3992 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1199 name=collection1 org.apache.solr.core.SolrCore@1065bc2 url=http://127.0.0.1:42026/collection1 node=127.0.0.1:42026_ C1199_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:42026, core=collection1, node_name=127.0.0.1:42026_}
   [junit4]   2> 802409 T3992 C1199 P42026 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:42026/collection1/
   [junit4]   2> 802409 T3948 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 802409 T3992 C1199 P42026 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 802410 T3992 C1199 P42026 oasc.SyncStrategy.syncToMe http://127.0.0.1:42026/collection1/ has no replicas
   [junit4]   2> 802410 T3992 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:42026/collection1/ shard1
   [junit4]   2> 802410 T3992 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 802411 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 802413 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 802415 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 802519 T3969 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 802519 T3987 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 802519 T3956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 802519 T3947 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 802564 T3992 oasc.ZkController.register We are http://127.0.0.1:42026/collection1/ and leader is http://127.0.0.1:42026/collection1/
   [junit4]   2> 802565 T3992 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:42026
   [junit4]   2> 802565 T3992 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 802565 T3992 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 802565 T3992 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 802567 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 802567 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 802567 T3947 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 802569 T3992 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 802569 T3948 oascc.ZkState

[...truncated too long message...]

30 oazc.StaticHostProvider.next WARN Unexpected exception java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]   2> 
   [junit4]   2> 964096 T4230 oaz.ClientCnxn$SendThread.run WARN Session 0x1467c37a3b80011 for server null, unexpected error, closing socket connection and attempting reconnect java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:739)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> 
   [junit4]   2> 964596 T4230 oazc.StaticHostProvider.next WARN Unexpected exception java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]   2> 
   [junit4]   2> 964597 T4230 oaz.ClientCnxn$SendThread.run WARN Session 0x1467c37a3b80011 for server null, unexpected error, closing socket connection and attempting reconnect java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:739)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> 
   [junit4]   2> 172013 T3924 ccr.ThreadLeakControl.tryToInterruptAll SEVERE There are still zombie threads that couldn't be terminated:
   [junit4]   2> 	   1) Thread[id=4230, name=TEST-BasicDistributedZkTest.testDistribSearch-seed#[B60A5BBE4920D73A]-SendThread(127.0.0.1:48844), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
   [junit4]   2> 	        at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]   2> 	        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]   2> 	        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]   2> NOTE: test params are: codec=Lucene41, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=pt, timezone=Europe/Volgograd
   [junit4]   2> NOTE: Linux 3.8.0-41-generic i386/Oracle Corporation 1.7.0_60 (32-bit)/cpus=8,threads=2,free=69366080,total=202600448
   [junit4]   2> NOTE: All tests run in this JVM: [TestImplicitCoreProperties, SolrIndexSplitterTest, TestSort, TriLevelCompositeIdRoutingTest, SoftAutoCommitTest, TestNonNRTOpen, DeleteShardTest, LukeRequestHandlerTest, TestSuggestSpellingConverter, ModifyConfFileTest, TestSolrXml, TestMaxScoreQueryParser, TestReloadAndDeleteDocs, DocumentBuilderTest, TestFreeTextSuggestions, DistributedTermsComponentTest, TestCustomSort, CircularListTest, SyncSliceTest, TestDefaultSimilarityFactory, TestAnalyzeInfixSuggestions, TestSearcherReuse, URLClassifyProcessorTest, TestFunctionQuery, TestSimpleQParserPlugin, TestSolrDeletionPolicy1, RAMDirectoryFactoryTest, JsonLoaderTest, TestReplicationHandlerBackup, TestSolrIndexConfig, DistributedQueryComponentCustomSortTest, OverseerCollectionProcessorTest, CacheHeaderTest, ChaosMonkeyNothingIsSafeTest, ExternalFileFieldSortTest, FieldMutatingUpdateProcessorTest, TestBlendedInfixSuggestions, BinaryUpdateRequestHandlerTest, TestReversedWildcardFilterFactory, CollectionsAPIAsyncDistributedZkTest, TestManagedSynonymFilterFactory, TestWriterPerf, CSVRequestHandlerTest, EchoParamsTest, TestDistributedMissingSort, DocValuesTest, SpatialFilterTest, PrimitiveFieldTypeTest, TestCloudManagedSchema, TestCloudManagedSchemaAddField, SliceStateUpdateTest, TestTrie, IndexSchemaTest, TestMergePolicyConfig, SolrCoreTest, TestDocSet, DistributedDebugComponentTest, TestClassNameShortening, TestInfoStreamLogging, TestQueryUtils, SuggesterTest, OverseerRolesTest, TestCharFilters, TestRealTimeGet, SpellPossibilityIteratorTest, TestLFUCache, TestDynamicFieldResource, SolrXmlInZkTest, TestCoreContainer, TestJmxMonitoredMap, TestBadConfig, DistributedExpandComponentTest, ReplicationFactorTest, SchemaVersionSpecificBehaviorTest, ClusterStateUpdateTest, ConnectionManagerTest, TestFieldSortValues, TestRandomMergePolicy, TestCollectionAPI, DistributedSuggestComponentTest, TestFaceting, DistribDocExpirationUpdateProcessorTest, TestSurroundQueryParser, TestReRankQParserPlugin, TestRandomDVFaceting, EnumFieldTest, TestGroupingSearch, TestDynamicFieldCollectionResource, ScriptEngineTest, UpdateParamsTest, LoggingHandlerTest, RegexBoostProcessorTest, TestLuceneMatchVersion, RecoveryZkTest, TestValueSourceCache, ShowFileRequestHandlerTest, SearchHandlerTest, BadComponentTest, PolyFieldTest, BasicDistributedZkTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=BasicDistributedZkTest -Dtests.seed=B60A5BBE4920D73A -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=pt -Dtests.timezone=Europe/Volgograd -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | BasicDistributedZkTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ERROR: SolrZkClient opens=18 closes=17
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B60A5BBE4920D73A]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.endTrackingZkClients(SolrTestCaseJ4.java:451)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:185)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest: 
   [junit4]    >    1) Thread[id=4231, name=TEST-BasicDistributedZkTest.testDistribSearch-seed#[B60A5BBE4920D73A]-EventThread, state=WAITING, group=TGRP-BasicDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   [junit4]    >    2) Thread[id=4230, name=TEST-BasicDistributedZkTest.testDistribSearch-seed#[B60A5BBE4920D73A]-SendThread(127.0.0.1:48844), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B60A5BBE4920D73A]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=4230, name=TEST-BasicDistributedZkTest.testDistribSearch-seed#[B60A5BBE4920D73A]-SendThread(127.0.0.1:48844), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B60A5BBE4920D73A]:0)
   [junit4] Completed on J0 in 172.07s, 1 test, 1 failure, 3 errors <<< FAILURES!

[...truncated 612 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 410 suites, 1693 tests, 3 suite-level errors, 1 error, 887 ignored (15 assumptions)

Total time: 56 minutes 32 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.7.0_60 -client -XX:+UseSerialGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.8.0_20-ea-b15) - Build # 10506 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10506/
Java: 32bit/jdk1.8.0_20-ea-b15 -server -XX:+UseConcMarkSweepGC

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ReplicationFactorTest

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([51B198A0607CEA91]:0)


REGRESSION:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([51B198A0607CEA91]:0)




Build Log:
[...truncated 11719 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/init-core-data-001
   [junit4]   2> 179294 T736 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 179298 T736 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 179299 T736 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 179299 T737 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 179399 T736 oasc.ZkTestServer.run start zk server on port:55628
   [junit4]   2> 179400 T736 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 179641 T743 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@17bcec2 name:ZooKeeperConnection Watcher:127.0.0.1:55628 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 179642 T736 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 179642 T736 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 179645 T736 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 179646 T745 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f6e617 name:ZooKeeperConnection Watcher:127.0.0.1:55628/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 179647 T736 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 179647 T736 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 179650 T736 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 179652 T736 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 179653 T736 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 179654 T736 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 179655 T736 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 179658 T736 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 179659 T736 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 179661 T736 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 179662 T736 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 179663 T736 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 179664 T736 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 179666 T736 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 179666 T736 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 179668 T736 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 179669 T736 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 179670 T736 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 179671 T736 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 179673 T736 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 179673 T736 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 179675 T736 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 179675 T736 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 179679 T736 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 179679 T736 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 179681 T736 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 179682 T736 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 179961 T736 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 179966 T736 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54632
   [junit4]   2> 179967 T736 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 179968 T736 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 179968 T736 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-002
   [junit4]   2> 179968 T736 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-002/'
   [junit4]   2> 179981 T736 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-002/solr.xml
   [junit4]   2> 179992 T736 oasc.CoreContainer.<init> New CoreContainer 10047213
   [junit4]   2> 179993 T736 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-002/]
   [junit4]   2> 179993 T736 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 179993 T736 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 179994 T736 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 179994 T736 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 179994 T736 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 179995 T736 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 179995 T736 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 179995 T736 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 179995 T736 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 179996 T736 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 179996 T736 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 179997 T736 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 179997 T736 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:55628/solr
   [junit4]   2> 179997 T736 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 179998 T736 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 180000 T756 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ba8960 name:ZooKeeperConnection Watcher:127.0.0.1:55628 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 180000 T736 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 180002 T736 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 180004 T758 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c0bc62 name:ZooKeeperConnection Watcher:127.0.0.1:55628/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 180004 T736 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 180005 T736 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 180007 T736 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 180009 T736 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 180016 T736 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 180020 T736 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 180022 T736 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 180023 T736 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52225_
   [junit4]   2> 180024 T736 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:52225_
   [junit4]   2> 180072 T736 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 180073 T736 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 180075 T736 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:52225_
   [junit4]   2> 180076 T736 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 180077 T736 oasc.Overseer.start Overseer (id=91896438760472579-127.0.0.1:52225_-n_0000000000) starting
   [junit4]   2> 180079 T736 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 180084 T760 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 180084 T736 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 180086 T736 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 180089 T736 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 180091 T759 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 180092 T761 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 180092 T761 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 180094 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 180094 T761 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 180095 T759 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 180095 T759 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52225",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52225_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 180095 T759 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 180095 T759 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 180097 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 180098 T758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 181094 T761 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 181094 T761 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 181095 T761 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 181095 T761 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 181096 T761 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 181096 T761 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-002/collection1/'
   [junit4]   2> 181097 T761 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 181097 T761 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 181097 T761 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 181117 T761 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 181127 T761 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 181128 T761 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 181132 T761 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 181224 T761 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 181229 T761 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 181231 T761 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 181243 T761 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 181246 T761 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 181249 T761 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 181250 T761 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 181250 T761 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 181250 T761 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 181251 T761 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 181251 T761 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 181251 T761 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 181252 T761 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 181252 T761 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-001/control/data/
   [junit4]   2> 181252 T761 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1fdb931
   [junit4]   2> 181255 T761 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-001/control/data
   [junit4]   2> 181255 T761 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-001/control/data/index/
   [junit4]   2> 181255 T761 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 181255 T761 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-001/control/data/index
   [junit4]   2> 181256 T761 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=20, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.44378209398457114]
   [junit4]   2> 181256 T761 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@18865f4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2875a4),segFN=segments_1,generation=1}
   [junit4]   2> 181256 T761 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 181258 T761 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 181259 T761 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 181259 T761 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 181259 T761 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 181259 T761 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 181259 T761 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 181262 T761 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 181262 T761 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 181262 T761 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 181263 T761 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 181263 T761 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 181263 T761 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 181263 T761 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 181264 T761 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 181264 T761 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 181264 T761 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 181264 T761 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 181266 T761 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 181267 T761 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 181268 T761 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 181268 T761 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=12, maxMergedSegmentMB=47.0830078125, floorSegmentMB=1.58984375, forceMergeDeletesPctAllowed=27.52163520959609, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 181269 T761 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@18865f4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2875a4),segFN=segments_1,generation=1}
   [junit4]   2> 181269 T761 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 181269 T761 oass.SolrIndexSearcher.<init> Opening Searcher@523baa[collection1] main
   [junit4]   2> 181269 T761 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 181270 T761 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 181270 T761 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 181270 T761 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 181270 T761 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 181271 T761 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 181271 T761 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 181271 T761 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 181271 T761 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 181272 T761 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 181274 T762 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@523baa[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 181275 T761 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 181276 T765 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:52225 collection:control_collection shard:shard1
   [junit4]   2> 181276 T736 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 181276 T736 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 181277 T765 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 181278 T736 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 181279 T768 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15f9c39 name:ZooKeeperConnection Watcher:127.0.0.1:55628/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 181279 T736 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 181280 T736 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 181283 T765 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 181283 T736 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 181284 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 181284 T765 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 181285 T765 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C807 name=collection1 org.apache.solr.core.SolrCore@1c48708 url=http://127.0.0.1:52225/collection1 node=127.0.0.1:52225_ C807_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:52225, core=collection1, node_name=127.0.0.1:52225_}
   [junit4]   2> 181285 T765 C807 P52225 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:52225/collection1/
   [junit4]   2> 181285 T759 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 181285 T765 C807 P52225 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 181285 T765 C807 P52225 oasc.SyncStrategy.syncToMe http://127.0.0.1:52225/collection1/ has no replicas
   [junit4]   2> 181286 T765 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:52225/collection1/ shard1
   [junit4]   2> 181286 T765 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 181292 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 181293 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 181296 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 181398 T758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 181398 T768 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 181445 T765 oasc.ZkController.register We are http://127.0.0.1:52225/collection1/ and leader is http://127.0.0.1:52225/collection1/
   [junit4]   2> 181445 T765 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:52225
   [junit4]   2> 181445 T765 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 181445 T765 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 181445 T765 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 181447 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 181447 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 181447 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 181447 T765 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 181448 T759 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 181449 T759 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52225",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52225_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 181450 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 181544 T736 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 181545 T736 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 181547 T736 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:55088
   [junit4]   2> 181547 T736 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 181548 T736 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 181548 T736 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-003
   [junit4]   2> 181548 T736 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-003/'
   [junit4]   2> 181552 T768 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 181552 T758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 181560 T736 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-003/solr.xml
   [junit4]   2> 181576 T736 oasc.CoreContainer.<init> New CoreContainer 6590436
   [junit4]   2> 181576 T736 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-003/]
   [junit4]   2> 181577 T736 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 181577 T736 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 181578 T736 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 181578 T736 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 181579 T736 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 181579 T736 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 181579 T736 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 181580 T736 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 181580 T736 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 181581 T736 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 181581 T736 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 181582 T736 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 181582 T736 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:55628/solr
   [junit4]   2> 181583 T736 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 181584 T736 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 181585 T779 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a73819 name:ZooKeeperConnection Watcher:127.0.0.1:55628 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 181586 T736 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 181587 T736 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 181594 T781 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c855a2 name:ZooKeeperConnection Watcher:127.0.0.1:55628/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 181594 T736 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 181601 T736 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 182604 T736 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45640_
   [junit4]   2> 182605 T736 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45640_
   [junit4]   2> 182611 T782 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 182611 T782 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 182612 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 182612 T782 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 182612 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 182612 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 182613 T759 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 182614 T759 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45640",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45640_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 182614 T759 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3]
   [junit4]   2> 182614 T759 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 182616 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 182720 T781 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 182720 T758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 182720 T768 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 183612 T782 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 183613 T782 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 183614 T782 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 183614 T782 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 183616 T782 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 183616 T782 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-003/collection1/'
   [junit4]   2> 183617 T782 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 183618 T782 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 183618 T782 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 183651 T782 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 183663 T782 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 183664 T782 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 183668 T782 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 183774 T782 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 183780 T782 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 183781 T782 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 183792 T782 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 183794 T782 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 183795 T782 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 183796 T782 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 183797 T782 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 183797 T782 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 183798 T782 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 183798 T782 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 183798 T782 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 183798 T782 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 183798 T782 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-001/jetty1/
   [junit4]   2> 183799 T782 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1fdb931
   [junit4]   2> 183799 T782 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-001/jetty1
   [junit4]   2> 183799 T782 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-001/jetty1/index/
   [junit4]   2> 183800 T782 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 183800 T782 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-001/jetty1/index
   [junit4]   2> 183800 T782 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=20, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.44378209398457114]
   [junit4]   2> 183801 T782 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@52d04e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1614f62),segFN=segments_1,generation=1}
   [junit4]   2> 183801 T782 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 183804 T782 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 183804 T782 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 183805 T782 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 183805 T782 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 183805 T782 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 183806 T782 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 183806 T782 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 183807 T782 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 183807 T782 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 183808 T782 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 183809 T782 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 183809 T782 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 183810 T782 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 183810 T782 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 183811 T782 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 183812 T782 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 183812 T782 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 183814 T782 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 183815 T782 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 183815 T782 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 183816 T782 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=12, maxMergedSegmentMB=47.0830078125, floorSegmentMB=1.58984375, forceMergeDeletesPctAllowed=27.52163520959609, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 183816 T782 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@52d04e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1614f62),segFN=segments_1,generation=1}
   [junit4]   2> 183816 T782 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 183817 T782 oass.SolrIndexSearcher.<init> Opening Searcher@195aba3[collection1] main
   [junit4]   2> 183817 T782 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 183818 T782 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 183818 T782 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 183819 T782 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 183819 T782 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 183819 T782 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 183820 T782 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 183820 T782 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 183820 T782 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 183820 T782 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 183822 T783 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@195aba3[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 183823 T782 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 183824 T786 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:45640 collection:collection1 shard:shard2
   [junit4]   2> 183824 T736 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 183825 T736 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 183825 T786 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 183829 T786 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 183830 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 183830 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 183830 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 183831 T786 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 183831 T786 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C808 name=collection1 org.apache.solr.core.SolrCore@b279e2 url=http://127.0.0.1:45640/collection1 node=127.0.0.1:45640_ C808_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:45640, core=collection1, node_name=127.0.0.1:45640_}
   [junit4]   2> 183831 T786 C808 P45640 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:45640/collection1/
   [junit4]   2> 183831 T786 C808 P45640 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 183831 T759 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 183831 T786 C808 P45640 oasc.SyncStrategy.syncToMe http://127.0.0.1:45640/collection1/ has no replicas
   [junit4]   2> 183832 T786 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:45640/collection1/ shard2
   [junit4]   2> 183832 T786 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 183833 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 183835 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 183838 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 183941 T781 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 183941 T758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 183941 T768 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 183986 T786 oasc.ZkController.register We are http://127.0.0.1:45640/collection1/ and leader is http://127.0.0.1:45640/collection1/
   [junit4]   2> 183986 T786 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:45640
   [junit4]   2> 183986 T786 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 183986 T786 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 183986 T786 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 183988 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 183988 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 183989 T786 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 183989 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 183991 T759 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 183991 T759 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45640",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45640_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 183993 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 184078 T736 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 184079 T736 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 184081 T736 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:36984
   [junit4]   2> 184082 T736 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 184082 T736 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 184083 T736 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-004
   [junit4]   2> 184086 T736 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-004/'
   [junit4]   2> 184095 T781 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 184095 T758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 184095 T768 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 184106 T736 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-004/solr.xml
   [junit4]   2> 184122 T736 oasc.CoreContainer.<init> New CoreContainer 15078497
   [junit4]   2> 184122 T736 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-004/]
   [junit4]   2> 184123 T736 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 184123 T736 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 184123 T736 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 184124 T736 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 184124 T736 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 184124 T736 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 184125 T736 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 184125 T736 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 184125 T736 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 184126 T736 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 184126 T736 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 184127 T736 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 184127 T736 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:55628/solr
   [junit4]   2> 184128 T736 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 184129 T736 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 184130 T798 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1784ae name:ZooKeeperConnection Watcher:127.0.0.1:55628 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 184131 T736 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 184134 T738 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1467b52fc710007, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 184134 T736 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 184135 T800 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1405f0a name:ZooKeeperConnection Watcher:127.0.0.1:55628/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 184136 T736 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 184143 T736 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 185145 T736 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44283_
   [junit4]   2> 185146 T736 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:44283_
   [junit4]   2> 185156 T801 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 185156 T801 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 185157 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 185157 T801 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 185157 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 185157 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 185157 T759 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 185158 T759 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44283",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:44283_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 185158 T759 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 185158 T759 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 185160 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 185262 T768 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 185263 T781 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 185262 T800 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 185262 T758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 186157 T801 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 186157 T801 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 186159 T801 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 186159 T801 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 186161 T801 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 186161 T801 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-004/collection1/'
   [junit4]   2> 186162 T801 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 186162 T801 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 186163 T801 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 186203 T801 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 186218 T801 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 186219 T801 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 186225 T801 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 186334 T801 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 186339 T801 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 186341 T801 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 186352 T801 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 186356 T801 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 186359 T801 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 186359 T801 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 186360 T801 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 186360 T801 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 186361 T801 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 186361 T801 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 186362 T801 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 186362 T801 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 186362 T801 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-001/jetty2/
   [junit4]   2> 186362 T801 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1fdb931
   [junit4]   2> 186363 T801 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-001/jetty2
   [junit4]   2> 186363 T801 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-001/jetty2/index/
   [junit4]   2> 186364 T801 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 186364 T801 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-001/jetty2/index
   [junit4]   2> 186364 T801 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=20, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.44378209398457114]
   [junit4]   2> 186365 T801 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@e69388 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@f74d26),segFN=segments_1,generation=1}
   [junit4]   2> 186365 T801 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 186369 T801 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 186370 T801 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 186370 T801 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 186370 T801 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 186371 T801 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 186371 T801 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 186371 T801 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 186372 T801 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 186372 T801 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 186372 T801 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 186373 T801 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 186373 T801 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 186373 T801 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 186374 T801 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 186374 T801 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 186375 T801 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 186375 T801 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 186377 T801 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 186379 T801 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 186379 T801 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 186380 T801 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=12, maxMergedSegmentMB=47.0830078125, floorSegmentMB=1.58984375, forceMergeDeletesPctAllowed=27.52163520959609, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 186380 T801 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@e69388 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@f74d26),segFN=segments_1,generation=1}
   [junit4]   2> 186381 T801 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 186381 T801 oass.SolrIndexSearcher.<init> Opening Searcher@b32254[collection1] main
   [junit4]   2> 186381 T801 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 186382 T801 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 186382 T801 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 186383 T801 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 186383 T801 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 186383 T801 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 186384 T801 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 186384 T801 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 186384 T801 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 186384 T801 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 186386 T802 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@b32254[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 186388 T801 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 186389 T805 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:44283 collection:collection1 shard:shard3
   [junit4]   2> 186389 T736 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 186390 T736 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 186390 T805 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 186394 T805 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 186396 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 186396 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 186396 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 186396 T805 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 186397 T805 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C809 name=collection1 org.apache.solr.core.SolrCore@2fd55f url=http://127.0.0.1:44283/collection1 node=127.0.0.1:44283_ C809_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:44283, core=collection1, node_name=127.0.0.1:44283_}
   [junit4]   2> 186397 T805 C809 P44283 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:44283/collection1/
   [junit4]   2> 186397 T759 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 186397 T805 C809 P44283 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 186398 T805 C809 P44283 oasc.SyncStrategy.syncToMe http://127.0.0.1:44283/collection1/ has no replicas
   [junit4]   2> 186398 T805 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:44283/collection1/ shard3
   [junit4]   2> 186398 T805 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 186400 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 186404 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 186406 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 186507 T768 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 186508 T758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 186507 T781 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 186507 T800 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 186555 T805 oasc.ZkController.register We are http://127.0.0.1:44283/collection1/ and leader is http://127.0.0.1:44283/collection1/
   [junit4]   2> 186555 T805 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:44283
   [junit4]   2> 186555 T805 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 186555 T805 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 186555 T805 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 186557 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 186557 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 186558 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 186558 T805 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 186558 T759 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 186559 T759 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44283",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:44283_",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 186560 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 186634 T736 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 186635 T736 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 186637 T736 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:46453
   [junit4]   2> 186637 T736 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 186638 T736 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 186638 T736 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-005
   [junit4]   2> 186638 T736 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-005/'
   [junit4]   2> 186648 T736 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-005/solr.xml
   [junit4]   2> 186662 T736 oasc.CoreContainer.<init> New CoreContainer 14061358
   [junit4]   2> 186663 T736 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-005/]
   [junit4]   2> 186663 T736 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 186663 T736 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 186664 T736 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 186664 T736 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 186664 T736 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 186664 T736 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 186665 T736 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 186665 T736 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 186665 T736 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 186666 T800 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 186666 T768 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 186666 T758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 186666 T781 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 186666 T736 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 186667 T736 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 186667 T736 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 186668 T736 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:55628/solr
   [junit4]   2> 186668 T736 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 186669 T736 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 186671 T817 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@166f1a9 name:ZooKeeperConnection Watcher:127.0.0.1:55628 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 186671 T736 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 186674 T736 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 186676 T819 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@977b69 name:ZooKeeperConnection Watcher:127.0.0.1:55628/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 186676 T736 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 186683 T736 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 187687 T736 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36456_
   [junit4]   2> 187689 T736 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:36456_
   [junit4]   2> 187699 T820 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 187700 T820 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 187701 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 187701 T820 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 187701 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 187702 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 187703 T759 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 187704 T759 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36456",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:36456_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 187704 T759 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 187704 T759 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 187709 T758 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 187814 T781 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 187814 T819 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 187814 T758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 187814 T768 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 187814 T800 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 188702 T820 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 188702 T820 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 188703 T820 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 188704 T820 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 188705 T820 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 188706 T820 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-005/collection1/'
   [junit4]   2> 188707 T820 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-005/collection1/lib/.svn/' to classloader
   [junit4]   2> 188707 T820 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-005/collection1/lib/classes/' to classloader
   [junit4]   2> 188708 T820 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest-51B198A0607CEA91-001/tempDir-005/collection1/lib/README' to classloader
   [junit4]   2> 188749 T820 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 188769 T820 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 188771 T820 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 188779 T820 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 188893 T820 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 188901 T820 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 188903 T820 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 188912 T820 oass.FileExchangeRateProvider.reload Reloading exc

[...truncated too long message...]

ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> 
   [junit4]   2> 7400999 T736 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 7401051 T736 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44283/
   [junit4]   2> 7401052 T736 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45640/
   [junit4]   2> 7401052 T736 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36456/
   [junit4]   2> 7401053 T736 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:52225/
   [junit4]   2> 7401080 T736 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 52225
   [junit4]   2> 7401081 T736 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 7401081 T736 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:55628 55628
   [junit4]   2> 7227588 T735 ccr.ThreadLeakControl.tryToInterruptAll SEVERE There are still zombie threads that couldn't be terminated:
   [junit4]   2> 	   1) Thread[id=736, name=TEST-ReplicationFactorTest.testDistribSearch-seed#[51B198A0607CEA91], state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]   2> 	        at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:343)
   [junit4]   2> 	        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2> 	        at java.net.Socket.connect(Socket.java:538)
   [junit4]   2> 	        at java.net.Socket.<init>(Socket.java:434)
   [junit4]   2> 	        at java.net.Socket.<init>(Socket.java:211)
   [junit4]   2> 	        at org.apache.solr.cloud.ZkTestServer.send4LetterWord(ZkTestServer.java:314)
   [junit4]   2> 	        at org.apache.solr.cloud.ZkTestServer.waitForServerDown(ZkTestServer.java:275)
   [junit4]   2> 	        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.shutdown(ZkTestServer.java:126)
   [junit4]   2> 	        at org.apache.solr.cloud.ZkTestServer.shutdown(ZkTestServer.java:262)
   [junit4]   2> 	        at org.apache.solr.cloud.AbstractDistribZkTestBase.tearDown(AbstractDistribZkTestBase.java:232)
   [junit4]   2> 	        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.tearDown(AbstractFullDistribZkTestBase.java:1427)
   [junit4]   2> 	        at org.apache.solr.cloud.ReplicationFactorTest.tearDown(ReplicationFactorTest.java:86)
   [junit4]   2> 	        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	        at java.lang.reflect.Method.invoke(Method.java:483)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:885)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
   [junit4]   2> 	        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
   [junit4]   2> 	        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 7406884 T736 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: test params are: codec=Lucene41, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=da_DK, timezone=Cuba
   [junit4]   2> NOTE: Linux 3.8.0-41-generic i386/Oracle Corporation 1.8.0_20-ea (32-bit)/cpus=8,threads=2,free=51127768,total=97882112
   [junit4]   2> NOTE: All tests run in this JVM: [TestStandardQParsers, TestModifyConfFiles, TestNRTOpen, UpdateParamsTest, BasicFunctionalityTest, TestConfigSets, TestBadConfig, UUIDFieldTest, HttpPartitionTest, TestRealTimeGet, HdfsBasicDistributedZkTest, TestCopyFieldCollectionResource, DistributedSpellCheckComponentTest, ReplicationFactorTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.seed=51B198A0607CEA91 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=da_DK -Dtests.timezone=Cuba -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | ReplicationFactorTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([51B198A0607CEA91]:0)
   [junit4] Completed on J0 in 7227.64s, 1 test, 2 errors <<< FAILURES!

[...truncated 271 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 410 suites, 1693 tests, 1 suite-level error, 1 error, 423 ignored (15 assumptions)

Total time: 164 minutes 29 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.8.0_20-ea-b15 -server -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (64bit/jdk1.7.0_60) - Build # 10505 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10505/
Java: 64bit/jdk1.7.0_60 -XX:-UseCompressedOops -XX:+UseG1GC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
shard1 is not consistent.  Got 391 from https://127.0.0.1:34430/bd/h/collection1lastClient and got 390 from https://127.0.0.1:41369/bd/h/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 391 from https://127.0.0.1:34430/bd/h/collection1lastClient and got 390 from https://127.0.0.1:41369/bd/h/collection1
	at __randomizedtesting.SeedInfo.seed([10BD5E22D9BC8F31:915BD03AAEE3EF0D]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1131)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1110)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:150)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11003 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/init-core-data-001
   [junit4]   2> 510616 T1946 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 510616 T1946 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /bd/h
   [junit4]   2> 510621 T1946 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 510621 T1946 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 510622 T1947 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 510722 T1946 oasc.ZkTestServer.run start zk server on port:49308
   [junit4]   2> 510723 T1946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 510742 T1953 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b029b4d name:ZooKeeperConnection Watcher:127.0.0.1:49308 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 510743 T1946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 510743 T1946 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 510747 T1946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 510748 T1955 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@cf1adcd name:ZooKeeperConnection Watcher:127.0.0.1:49308/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 510748 T1946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 510749 T1946 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 510752 T1946 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 510755 T1946 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 510756 T1946 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 510758 T1946 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 510758 T1946 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 510761 T1946 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 510762 T1946 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 510764 T1946 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 510764 T1946 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 510767 T1946 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 510767 T1946 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 510769 T1946 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 510769 T1946 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 510771 T1946 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 510771 T1946 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 510773 T1946 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 510773 T1946 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 510775 T1946 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 510775 T1946 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 510777 T1946 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 510777 T1946 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 510779 T1946 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 510779 T1946 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 510782 T1946 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 510782 T1946 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 510788 T1946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 510789 T1957 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14e13ba6 name:ZooKeeperConnection Watcher:127.0.0.1:49308/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 510789 T1946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 510999 T1946 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 511002 T1946 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 511004 T1946 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:55801
   [junit4]   2> 511006 T1946 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 511006 T1946 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 511006 T1946 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-002
   [junit4]   2> 511006 T1946 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-002/'
   [junit4]   2> 511018 T1946 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-002/solr.xml
   [junit4]   2> 511042 T1946 oasc.CoreContainer.<init> New CoreContainer 150454495
   [junit4]   2> 511043 T1946 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-002/]
   [junit4]   2> 511043 T1946 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 511044 T1946 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 511044 T1946 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 511044 T1946 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 511044 T1946 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 511045 T1946 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 511045 T1946 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 511045 T1946 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 511045 T1946 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 511047 T1946 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 511047 T1946 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 511048 T1946 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 511048 T1946 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49308/solr
   [junit4]   2> 511048 T1946 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 511049 T1946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 511050 T1968 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@54257467 name:ZooKeeperConnection Watcher:127.0.0.1:49308 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 511051 T1946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 511053 T1946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 511053 T1970 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5350c62a name:ZooKeeperConnection Watcher:127.0.0.1:49308/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 511054 T1946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 511055 T1946 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 511057 T1946 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 511060 T1946 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 511061 T1946 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 511063 T1946 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 511065 T1946 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 511066 T1946 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55801_bd%2Fh
   [junit4]   2> 511067 T1946 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55801_bd%2Fh
   [junit4]   2> 511069 T1946 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 511070 T1946 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 511074 T1946 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:55801_bd%2Fh
   [junit4]   2> 511074 T1946 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 511076 T1946 oasc.Overseer.start Overseer (id=91895988142080004-127.0.0.1:55801_bd%2Fh-n_0000000000) starting
   [junit4]   2> 511078 T1946 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 511085 T1972 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 511085 T1946 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 511087 T1946 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 511089 T1946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 511092 T1971 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 511097 T1973 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 511097 T1973 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 511099 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 511099 T1973 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 511101 T1971 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 511102 T1971 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:55801/bd/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:55801_bd%2Fh",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 511103 T1971 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 511103 T1971 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 511105 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 511109 T1970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 512100 T1973 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 512100 T1973 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 512101 T1973 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 512101 T1973 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 512102 T1973 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 512102 T1973 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-002/collection1/'
   [junit4]   2> 512102 T1973 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 512103 T1973 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 512103 T1973 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 512131 T1973 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 512156 T1973 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 512158 T1973 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 512163 T1973 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 512367 T1973 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 512369 T1973 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 512371 T1973 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 512374 T1973 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 512389 T1973 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 512389 T1973 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 512389 T1973 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/control/data/
   [junit4]   2> 512389 T1973 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2af604de
   [junit4]   2> 512390 T1973 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/control/data
   [junit4]   2> 512390 T1973 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/control/data/index/
   [junit4]   2> 512390 T1973 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 512391 T1973 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/control/data/index
   [junit4]   2> 512391 T1973 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1568285115, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 512395 T1973 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 512395 T1973 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 512399 T1973 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 512399 T1973 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 512400 T1973 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 512400 T1973 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 512400 T1973 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 512400 T1973 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 512400 T1973 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 512401 T1973 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 512401 T1973 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 512401 T1973 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 512402 T1973 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 512402 T1973 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 512402 T1973 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 512402 T1973 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 512403 T1973 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 512403 T1973 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 512403 T1973 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 512406 T1973 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 512407 T1973 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 512408 T1973 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 512408 T1973 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=44, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 512409 T1973 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 512409 T1973 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 512410 T1973 oass.SolrIndexSearcher.<init> Opening Searcher@1dbe03a0[collection1] main
   [junit4]   2> 512410 T1973 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 512411 T1973 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 512411 T1973 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 512412 T1973 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 512412 T1973 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 512412 T1973 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 512412 T1973 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 512413 T1973 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 512413 T1973 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 512413 T1973 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 512415 T1974 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1dbe03a0[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 512416 T1973 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 512417 T1977 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:55801/bd/h collection:control_collection shard:shard1
   [junit4]   2> 512417 T1946 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 512417 T1946 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 512418 T1977 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 512420 T1946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 512423 T1979 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@77d71772 name:ZooKeeperConnection Watcher:127.0.0.1:49308/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 512424 T1946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 512425 T1946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 512425 T1977 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 512427 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 512427 T1977 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 512427 T1977 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C898 name=collection1 org.apache.solr.core.SolrCore@4c45fa03 url=https://127.0.0.1:55801/bd/h/collection1 node=127.0.0.1:55801_bd%2Fh C898_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:55801/bd/h, core=collection1, node_name=127.0.0.1:55801_bd%2Fh}
   [junit4]   2> 512427 T1977 C898 P55801 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:55801/bd/h/collection1/
   [junit4]   2> 512427 T1977 C898 P55801 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 512428 T1977 C898 P55801 oasc.SyncStrategy.syncToMe https://127.0.0.1:55801/bd/h/collection1/ has no replicas
   [junit4]   2> 512428 T1971 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 512428 T1977 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:55801/bd/h/collection1/ shard1
   [junit4]   2> 512428 T1946 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 512428 T1977 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 512430 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 512432 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 512434 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 512536 T1979 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 512536 T1970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 512583 T1977 oasc.ZkController.register We are https://127.0.0.1:55801/bd/h/collection1/ and leader is https://127.0.0.1:55801/bd/h/collection1/
   [junit4]   2> 512583 T1977 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:55801/bd/h
   [junit4]   2> 512583 T1977 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 512584 T1977 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 512584 T1977 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 512585 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 512585 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 512586 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 512586 T1977 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 512587 T1971 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 512587 T1971 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:55801/bd/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:55801_bd%2Fh",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 512591 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 512655 T1946 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 512655 T1946 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 512658 T1946 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 512659 T1946 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:45723
   [junit4]   2> 512662 T1946 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 512662 T1946 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 512662 T1946 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-003
   [junit4]   2> 512663 T1946 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-003/'
   [junit4]   2> 512675 T1946 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-003/solr.xml
   [junit4]   2> 512694 T1970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 512694 T1979 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 512702 T1946 oasc.CoreContainer.<init> New CoreContainer 1466523309
   [junit4]   2> 512702 T1946 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-003/]
   [junit4]   2> 512703 T1946 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 512703 T1946 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 512703 T1946 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 512703 T1946 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 512704 T1946 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 512704 T1946 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 512704 T1946 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 512704 T1946 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 512705 T1946 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 512707 T1946 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 512707 T1946 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 512707 T1946 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 512708 T1946 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49308/solr
   [junit4]   2> 512708 T1946 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 512709 T1946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 512710 T1990 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@77deaccd name:ZooKeeperConnection Watcher:127.0.0.1:49308 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 512710 T1946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 512712 T1946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 512714 T1992 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@770535bf name:ZooKeeperConnection Watcher:127.0.0.1:49308/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 512714 T1946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 512718 T1946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 513721 T1946 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45723_bd%2Fh
   [junit4]   2> 513723 T1946 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45723_bd%2Fh
   [junit4]   2> 513731 T1993 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 513732 T1993 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 513733 T1993 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 513733 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 513733 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 513733 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 513734 T1971 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 513735 T1971 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:45723/bd/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45723_bd%2Fh",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 513735 T1971 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3]
   [junit4]   2> 513735 T1971 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 513736 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 513838 T1979 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 513838 T1992 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 513838 T1970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 514733 T1993 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 514733 T1993 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 514734 T1993 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 514734 T1993 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 514735 T1993 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 514735 T1993 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-003/collection1/'
   [junit4]   2> 514735 T1993 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 514736 T1993 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 514736 T1993 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 514764 T1993 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 514788 T1993 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 514790 T1993 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 514795 T1993 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 515000 T1993 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 515002 T1993 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 515003 T1993 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 515007 T1993 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 515023 T1993 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 515024 T1993 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 515024 T1993 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/jetty1/
   [junit4]   2> 515024 T1993 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2af604de
   [junit4]   2> 515025 T1993 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/jetty1
   [junit4]   2> 515025 T1993 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/jetty1/index/
   [junit4]   2> 515025 T1993 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 515026 T1993 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/jetty1/index
   [junit4]   2> 515026 T1993 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1568285115, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 515031 T1993 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 515032 T1993 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 515036 T1993 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 515036 T1993 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 515036 T1993 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 515036 T1993 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 515037 T1993 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 515037 T1993 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 515037 T1993 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 515037 T1993 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 515038 T1993 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 515038 T1993 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 515038 T1993 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 515039 T1993 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 515039 T1993 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 515039 T1993 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 515040 T1993 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 515040 T1993 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 515040 T1993 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 515043 T1993 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 515044 T1993 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 515044 T1993 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 515045 T1993 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=44, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 515046 T1993 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 515046 T1993 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 515047 T1993 oass.SolrIndexSearcher.<init> Opening Searcher@36f99008[collection1] main
   [junit4]   2> 515047 T1993 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 515048 T1993 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 515048 T1993 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 515048 T1993 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 515048 T1993 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 515049 T1993 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 515049 T1993 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 515049 T1993 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 515050 T1993 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 515050 T1993 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 515052 T1994 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@36f99008[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 515055 T1993 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 515055 T1997 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:45723/bd/h collection:collection1 shard:shard3
   [junit4]   2> 515056 T1946 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 515056 T1946 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 515056 T1997 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 515061 T1997 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 515062 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 515062 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 515062 T1997 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 515062 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 515062 T1997 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C899 name=collection1 org.apache.solr.core.SolrCore@50b5967c url=https://127.0.0.1:45723/bd/h/collection1 node=127.0.0.1:45723_bd%2Fh C899_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:45723/bd/h, core=collection1, node_name=127.0.0.1:45723_bd%2Fh}
   [junit4]   2> 515063 T1997 C899 P45723 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:45723/bd/h/collection1/
   [junit4]   2> 515063 T1997 C899 P45723 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 515063 T1971 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 515063 T1997 C899 P45723 oasc.SyncStrategy.syncToMe https://127.0.0.1:45723/bd/h/collection1/ has no replicas
   [junit4]   2> 515063 T1997 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:45723/bd/h/collection1/ shard3
   [junit4]   2> 515064 T1997 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 515065 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 515067 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 515069 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 515172 T1970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 515172 T1979 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 515172 T1992 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 515218 T1997 oasc.ZkController.register We are https://127.0.0.1:45723/bd/h/collection1/ and leader is https://127.0.0.1:45723/bd/h/collection1/
   [junit4]   2> 515219 T1997 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:45723/bd/h
   [junit4]   2> 515219 T1997 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 515219 T1997 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 515219 T1997 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 515221 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 515221 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 515221 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 515222 T1997 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 515222 T1971 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 515223 T1971 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:45723/bd/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45723_bd%2Fh",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 515225 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 515269 T1946 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 515270 T1946 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 515272 T1946 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 515273 T1946 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:34430
   [junit4]   2> 515276 T1946 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 515276 T1946 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 515276 T1946 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-004
   [junit4]   2> 515277 T1946 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-004/'
   [junit4]   2> 515289 T1946 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-004/solr.xml
   [junit4]   2> 515322 T1946 oasc.CoreContainer.<init> New CoreContainer 580453601
   [junit4]   2> 515323 T1946 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-004/]
   [junit4]   2> 515324 T1946 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 515324 T1946 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 515324 T1946 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 515324 T1946 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 515324 T1946 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 515325 T1946 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 515325 T1946 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 515325 T1946 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 515325 T1946 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 515328 T1946 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 515328 T1946 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 515328 T1946 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 515328 T1946 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49308/solr
   [junit4]   2> 515329 T1946 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 515329 T1946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 515330 T1979 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 515330 T1970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 515330 T1992 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 515331 T2008 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@168cbd28 name:ZooKeeperConnection Watcher:127.0.0.1:49308 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 515332 T1946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 515334 T1946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 515335 T2010 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49d94011 name:ZooKeeperConnection Watcher:127.0.0.1:49308/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 515335 T1946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 515340 T1946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 516343 T1946 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34430_bd%2Fh
   [junit4]   2> 516344 T1946 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:34430_bd%2Fh
   [junit4]   2> 516352 T2011 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 516353 T2011 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 516354 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 516354 T2011 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 516354 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 516354 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 516355 T1971 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 516356 T1971 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:34430/bd/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34430_bd%2Fh",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 516356 T1971 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 516356 T1971 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 516358 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 516460 T2010 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 516461 T1970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 516460 T1992 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 516460 T1979 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 517354 T2011 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 517354 T2011 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 517355 T2011 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 517355 T2011 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 517356 T2011 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 517356 T2011 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-004/collection1/'
   [junit4]   2> 517357 T2011 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 517357 T2011 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 517358 T2011 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 517388 T2011 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 517462 T2011 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 517464 T2011 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 517470 T2011 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 517687 T2011 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 517689 T2011 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 517690 T2011 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 517694 T2011 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 517725 T2011 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 517725 T2011 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 517725 T2011 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/jetty2/
   [junit4]   2> 517725 T2011 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2af604de
   [junit4]   2> 517726 T2011 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/jetty2
   [junit4]   2> 517726 T2011 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/jetty2/index/
   [junit4]   2> 517726 T2011 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 517727 T2011 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/jetty2/index
   [junit4]   2> 517728 T2011 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1568285115, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 517731 T2011 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/jetty2/index,segFN=segments_1,generation=1}
   [junit4]   2> 517731 T2011 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 517736 T2011 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 517737 T2011 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 517737 T2011 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 517737 T2011 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 517737 T2011 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 517738 T2011 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 517738 T2011 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 517738 T2011 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 517738 T2011 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 517739 T2011 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 517740 T2011 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 517740 T2011 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 517740 T2011 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 517741 T2011 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 517741 T2011 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 517742 T2011 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 517742 T2011 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 517744 T2011 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 517747 T2011 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 517747 T2011 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 517748 T2011 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=44, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 517749 T2011 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/jetty2/index,segFN=segments_1,generation=1}
   [junit4]   2> 517749 T2011 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 517749 T2011 oass.SolrIndexSearcher.<init> Opening Searcher@73dac261[collection1] main
   [junit4]   2> 517750 T2011 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 517750 T2011 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 517751 T2011 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 517751 T2011 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 517751 T2011 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 517752 T2011 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 517752 T2011 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 517752 T2011 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 517753 T2011 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 517753 T2011 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 517756 T2012 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@73dac261[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 517759 T2011 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 517759 T2015 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:34430/bd/h collection:collection1 shard:shard1
   [junit4]   2> 517760 T1946 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 517760 T1946 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 517760 T2015 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 517765 T2015 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 517766 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 517767 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 517767 T2015 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 517767 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 517767 T2015 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C900 name=collection1 org.apache.solr.core.SolrCore@738dadcf url=https://127.0.0.1:34430/bd/h/collection1 node=127.0.0.1:34430_bd%2Fh C900_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:34430/bd/h, core=collection1, node_name=127.0.0.1:34430_bd%2Fh}
   [junit4]   2> 517767 T2015 C900 P34430 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:34430/bd/h/collection1/
   [junit4]   2> 517768 T2015 C900 P34430 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 517768 T2015 C900 P34430 oasc.SyncStrategy.syncToMe https://127.0.0.1:34430/bd/h/collection1/ has no replicas
   [junit4]   2> 517768 T2015 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:34430/bd/h/collection1/ shard1
   [junit4]   2> 517768 T2015 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 517769 T1971 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 517775 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 517777 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 517779 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 517881 T1979 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 517881 T2010 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 517881 T1970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 517881 T1992 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 517931 T2015 oasc.ZkController.register We are https://127.0.0.1:34430/bd/h/collection1/ and leader is https://127.0.0.1:34430/bd/h/collection1/
   [junit4]   2> 517931 T2015 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:34430/bd/h
   [junit4]   2> 517932 T2015 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 517932 T2015 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 517932 T2015 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 517934 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 517935 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 517935 T2015 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 517935 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 517938 T1971 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 517939 T1971 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:34430/bd/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34430_bd%2Fh",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 517941 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 518044 T2010 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 518044 T1970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 518044 T1979 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 518044 T1992 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 518070 T1946 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 518071 T1946 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 518075 T1946 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 518076 T1946 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:50919
   [junit4]   2> 518078 T1946 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 518079 T1946 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 518079 T1946 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-005
   [junit4]   2> 518079 T1946 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-005/'
   [junit4]   2> 518098 T1946 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-005/solr.xml
   [junit4]   2> 518170 T1946 oasc.CoreContainer.<init> New CoreContainer 781197366
   [junit4]   2> 518170 T1946 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-005/]
   [junit4]   2> 518171 T1946 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 518172 T1946 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 518172 T1946 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 518172 T1946 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 518173 T1946 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 518173 T1946 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 518174 T1946 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 518174 T1946 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 518174 T1946 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 518177 T1946 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 518178 T1946 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 518178 T1946 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 518178 T1946 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49308/solr
   [junit4]   2> 518179 T1946 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 518181 T1946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 518182 T2026 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@475b9c76 name:ZooKeeperConnection Watcher:127.0.0.1:49308 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 518183 T1946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 518186 T1946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 518188 T2028 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@39f89ef0 name:ZooKeeperConnection Watcher:127.0.0.1:49308/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 518189 T1946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 518202 T1946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 519205 T1946 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50919_bd%2Fh
   [junit4]   2> 519206 T1946 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50919_bd%2Fh
   [junit4]   2> 519215 T2029 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 519215 T2029 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 519216 T1970 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 519216 T2029 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 519216 T1970 oasc

[...truncated too long message...]

ve nodes size: 10)
   [junit4]   2> 653699 T2682 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001/tempDir-001/jetty12/index.20140608035946519
   [junit4]   2> 653699 T2909 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 653699 T2682 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 653699 T2987 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 653700 T2682 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 653700 T2909 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 12)
   [junit4]   2> 653700 T2987 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 653701 T2909 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 653701 T2987 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 653701 T2909 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.method=testDistribSearch -Dtests.seed=10BD5E22D9BC8F31 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es_PR -Dtests.timezone=Canada/Mountain -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE  143s J1 | ChaosMonkeySafeLeaderTest.testDistribSearch <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: shard1 is not consistent.  Got 391 from https://127.0.0.1:34430/bd/h/collection1lastClient and got 390 from https://127.0.0.1:41369/bd/h/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([10BD5E22D9BC8F31:915BD03AAEE3EF0D]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1131)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1110)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:150)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 653709 T1946 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.ChaosMonkeySafeLeaderTest-10BD5E22D9BC8F31-001
   [junit4]   2> NOTE: test params are: codec=Lucene49: {a_i=PostingsFormat(name=MockRandom), _version_=Lucene41(blocksize=128), id=PostingsFormat(name=MockRandom), rnd_b=PostingsFormat(name=MockRandom), a_t=Pulsing41(freqCutoff=8 minBlockSize=7 maxBlockSize=62)}, docValues:{}, sim=DefaultSimilarity, locale=es_PR, timezone=Canada/Mountain
   [junit4]   2> NOTE: Linux 3.8.0-41-generic amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=8,threads=1,free=141743760,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [OpenCloseCoreStressTest, RequiredFieldsTest, DeleteReplicaTest, TestSchemaSimilarityResource, TestHighlightDedupGrouping, TestCSVResponseWriter, SolrIndexConfigTest, SpellCheckComponentTest, DocExpirationUpdateProcessorFactoryTest, CoreAdminRequestStatusTest, TestLMDirichletSimilarityFactory, DisMaxRequestHandlerTest, SuggestComponentTest, DateMathParserTest, TestStressLucene, CoreMergeIndexesAdminHandlerTest, TestManagedSchemaFieldResource, HdfsLockFactoryTest, InfoHandlerTest, SSLMigrationTest, HighlighterConfigTest, ShardSplitTest, DistanceFunctionTest, TestLRUCache, ShardRoutingCustomTest, TestDistributedGrouping, CurrencyFieldXmlFileTest, StandardRequestHandlerTest, PreAnalyzedUpdateProcessorTest, BlockCacheTest, NumericFieldsTest, TestStressRecovery, TestCollationField, BJQParserTest, AnalysisErrorHandlingTest, TestStressUserVersions, TestSchemaResource, PrimUtilsTest, QueryResultKeyTest, TestFieldTypeCollectionResource, FullSolrCloudDistribCmdsTest, SortByFunctionTest, NoFacetTest, TestPartialUpdateDeduplication, TestArbitraryIndexDir, SolrCmdDistributorTest, HdfsBasicDistributedZkTest, TestSolrCoreProperties, TestHashPartitioner, SystemInfoHandlerTest, SolrCoreCheckLockOnStartupTest, SuggesterWFSTTest, TestRecovery, CoreAdminCreateDiscoverTest, ResponseHeaderTest, TestSolrJ, ChaosMonkeySafeLeaderTest]
   [junit4] Completed on J1 in 143.12s, 1 test, 1 failure <<< FAILURES!

[...truncated 821 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 410 suites, 1693 tests, 1 failure, 36 ignored (15 assumptions)

Total time: 65 minutes 56 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0_60 -XX:-UseCompressedOops -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.8.0_20-ea-b15) - Build # 10504 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10504/
Java: 32bit/jdk1.8.0_20-ea-b15 -server -XX:+UseConcMarkSweepGC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch

Error Message:
commitWithin did not work expected:<68> but was:<67>

Stack Trace:
java.lang.AssertionError: commitWithin did not work expected:<68> but was:<67>
	at __randomizedtesting.SeedInfo.seed([B98D7890D2A4D556:386BF688A5FBB56A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.cloud.BasicDistributedZkTest.doTest(BasicDistributedZkTest.java:347)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11146 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZkTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/init-core-data-001
   [junit4]   2> 460648 T2200 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 460651 T2200 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 460651 T2200 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 460652 T2201 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 460752 T2200 oasc.ZkTestServer.run start zk server on port:60165
   [junit4]   2> 460753 T2200 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 460849 T2207 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@28b6b7 name:ZooKeeperConnection Watcher:127.0.0.1:60165 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 460850 T2200 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 460850 T2200 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 460857 T2200 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 460858 T2209 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1bde7b1 name:ZooKeeperConnection Watcher:127.0.0.1:60165/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 460858 T2200 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 460859 T2200 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 460860 T2200 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 460861 T2200 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 460863 T2200 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 460864 T2200 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 460864 T2200 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 460866 T2200 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 460867 T2200 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 460868 T2200 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 460869 T2200 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 460870 T2200 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 460870 T2200 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 460873 T2200 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 460874 T2200 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 460876 T2200 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 460876 T2200 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 460878 T2200 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 460878 T2200 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 460881 T2200 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 460882 T2200 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 460884 T2200 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 460885 T2200 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 460887 T2200 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 460887 T2200 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 460889 T2200 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 460889 T2200 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 461165 T2200 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 461171 T2200 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:45094
   [junit4]   2> 461172 T2200 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 461172 T2200 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 461173 T2200 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-002
   [junit4]   2> 461173 T2200 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-002/'
   [junit4]   2> 461184 T2200 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-002/solr.xml
   [junit4]   2> 461198 T2200 oasc.CoreContainer.<init> New CoreContainer 1228167
   [junit4]   2> 461198 T2200 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-002/]
   [junit4]   2> 461199 T2200 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 461199 T2200 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 461199 T2200 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 461199 T2200 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 461200 T2200 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 461200 T2200 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 461200 T2200 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 461201 T2200 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 461201 T2200 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 461201 T2200 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 461202 T2200 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 461202 T2200 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 461202 T2200 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:60165/solr
   [junit4]   2> 461203 T2200 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 461203 T2200 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 461205 T2220 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b1ede4 name:ZooKeeperConnection Watcher:127.0.0.1:60165 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 461205 T2200 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 461206 T2200 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 461209 T2222 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@97c6ab name:ZooKeeperConnection Watcher:127.0.0.1:60165/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 461209 T2200 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 461210 T2200 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 461212 T2200 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 461214 T2200 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 461216 T2200 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 461218 T2200 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 461220 T2200 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 461222 T2200 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45094_
   [junit4]   2> 461223 T2200 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45094_
   [junit4]   2> 461229 T2200 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 461230 T2200 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 461232 T2200 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:45094_
   [junit4]   2> 461232 T2200 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 461234 T2200 oasc.Overseer.start Overseer (id=91895327484280835-127.0.0.1:45094_-n_0000000000) starting
   [junit4]   2> 461235 T2200 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 461240 T2224 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 461241 T2200 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 461242 T2200 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 461243 T2200 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 461244 T2223 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 461246 T2225 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 461246 T2225 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 461247 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 461247 T2225 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 461247 T2223 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 461248 T2223 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45094",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45094_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 461248 T2223 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 461248 T2223 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 461251 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 461252 T2222 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 462247 T2225 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 462247 T2225 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 462248 T2225 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 462248 T2225 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 462249 T2225 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 462249 T2225 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-002/collection1/'
   [junit4]   2> 462250 T2225 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 462250 T2225 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 462250 T2225 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 462267 T2225 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 462275 T2225 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 462276 T2225 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 462279 T2225 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 462360 T2225 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 462363 T2225 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 462364 T2225 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 462371 T2225 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 462373 T2225 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 462375 T2225 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 462376 T2225 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 462376 T2225 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 462377 T2225 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 462377 T2225 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 462377 T2225 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 462378 T2225 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 462378 T2225 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 462378 T2225 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/control/data/
   [junit4]   2> 462378 T2225 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ac36c0
   [junit4]   2> 462379 T2225 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/control/data
   [junit4]   2> 462379 T2225 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/control/data/index/
   [junit4]   2> 462379 T2225 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 462380 T2225 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/control/data/index
   [junit4]   2> 462380 T2225 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=18.86328125, floorSegmentMB=0.591796875, forceMergeDeletesPctAllowed=0.1200649045530755, segmentsPerTier=8.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2075689064913826
   [junit4]   2> 462381 T2225 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7ccfe5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7ec781),segFN=segments_1,generation=1}
   [junit4]   2> 462381 T2225 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 462383 T2225 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 462384 T2225 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 462384 T2225 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 462384 T2225 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 462384 T2225 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 462385 T2225 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 462385 T2225 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 462385 T2225 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 462385 T2225 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 462386 T2225 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 462386 T2225 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 462386 T2225 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 462386 T2225 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 462386 T2225 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 462387 T2225 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 462387 T2225 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 462387 T2225 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 462389 T2225 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 462391 T2225 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 462391 T2225 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 462392 T2225 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 462392 T2225 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7ccfe5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7ec781),segFN=segments_1,generation=1}
   [junit4]   2> 462392 T2225 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 462393 T2225 oass.SolrIndexSearcher.<init> Opening Searcher@1bb9541[collection1] main
   [junit4]   2> 462393 T2225 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 462395 T2225 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 462396 T2225 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 462396 T2225 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 462396 T2225 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 462397 T2225 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 462397 T2225 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 462397 T2225 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 462397 T2225 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 462398 T2225 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 462399 T2226 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1bb9541[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 462400 T2225 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 462401 T2229 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:45094 collection:control_collection shard:shard1
   [junit4]   2> 462401 T2200 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 462402 T2200 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 462402 T2229 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 462403 T2200 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 462406 T2231 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f0bf3e name:ZooKeeperConnection Watcher:127.0.0.1:60165/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 462406 T2200 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 462408 T2200 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 462410 T2200 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 462410 T2229 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 462414 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 462415 T2229 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 462415 T2229 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 462415 T2223 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C1890 name=collection1 org.apache.solr.core.SolrCore@75af36 url=http://127.0.0.1:45094/collection1 node=127.0.0.1:45094_ C1890_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:45094, core=collection1, node_name=127.0.0.1:45094_}
   [junit4]   2> 462416 T2229 C1890 P45094 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:45094/collection1/
   [junit4]   2> 462416 T2229 C1890 P45094 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 462417 T2229 C1890 P45094 oasc.SyncStrategy.syncToMe http://127.0.0.1:45094/collection1/ has no replicas
   [junit4]   2> 462417 T2229 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:45094/collection1/ shard1
   [junit4]   2> 462417 T2229 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 462418 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 462422 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 462425 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 462526 T2231 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 462526 T2222 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 462574 T2229 oasc.ZkController.register We are http://127.0.0.1:45094/collection1/ and leader is http://127.0.0.1:45094/collection1/
   [junit4]   2> 462574 T2229 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:45094
   [junit4]   2> 462574 T2229 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 462574 T2229 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 462575 T2229 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 462576 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 462577 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 462577 T2229 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 462577 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 462578 T2223 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 462578 T2223 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45094",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45094_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 462579 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 462667 T2200 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 462667 T2200 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 462675 T2200 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:48570
   [junit4]   2> 462678 T2200 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 462678 T2200 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 462679 T2200 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-003
   [junit4]   2> 462679 T2200 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-003/'
   [junit4]   2> 462681 T2231 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 462681 T2222 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 462690 T2200 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-003/solr.xml
   [junit4]   2> 462697 T2200 oasc.CoreContainer.<init> New CoreContainer 14109353
   [junit4]   2> 462697 T2200 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-003/]
   [junit4]   2> 462698 T2200 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 462698 T2200 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 462699 T2200 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 462699 T2200 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 462699 T2200 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 462699 T2200 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 462700 T2200 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 462700 T2200 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 462700 T2200 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 462701 T2200 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 462701 T2200 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 462701 T2200 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 462702 T2200 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:60165/solr
   [junit4]   2> 462702 T2200 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 462703 T2200 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 462704 T2242 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@104817 name:ZooKeeperConnection Watcher:127.0.0.1:60165 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 462704 T2200 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 462710 T2200 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 462711 T2244 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@607a46 name:ZooKeeperConnection Watcher:127.0.0.1:60165/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 462711 T2200 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 462714 T2200 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 463716 T2200 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48570_
   [junit4]   2> 463717 T2200 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48570_
   [junit4]   2> 463722 T2245 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 463722 T2245 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 463723 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 463723 T2245 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 463723 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 463723 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 463724 T2223 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 463724 T2223 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48570",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:48570_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 463725 T2223 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 463725 T2223 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 463726 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 463828 T2222 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 463828 T2231 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 463828 T2244 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 464723 T2245 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 464724 T2245 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 464724 T2245 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 464724 T2245 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 464725 T2245 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 464725 T2245 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-003/collection1/'
   [junit4]   2> 464726 T2245 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 464726 T2245 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 464726 T2245 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 464743 T2245 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 464751 T2245 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 464752 T2245 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 464756 T2245 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 464835 T2245 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 464839 T2245 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 464840 T2245 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 464847 T2245 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 464849 T2245 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 464851 T2245 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 464852 T2245 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 464852 T2245 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 464852 T2245 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 464853 T2245 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 464854 T2245 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 464854 T2245 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 464854 T2245 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 464854 T2245 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty1/
   [junit4]   2> 464855 T2245 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ac36c0
   [junit4]   2> 464855 T2245 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty1
   [junit4]   2> 464856 T2245 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty1/index/
   [junit4]   2> 464856 T2245 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 464856 T2245 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty1/index
   [junit4]   2> 464857 T2245 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=18.86328125, floorSegmentMB=0.591796875, forceMergeDeletesPctAllowed=0.1200649045530755, segmentsPerTier=8.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2075689064913826
   [junit4]   2> 464857 T2245 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@8a386f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@af05de),segFN=segments_1,generation=1}
   [junit4]   2> 464858 T2245 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 464861 T2245 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 464861 T2245 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 464862 T2245 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 464862 T2245 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 464862 T2245 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 464862 T2245 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 464862 T2245 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 464863 T2245 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 464863 T2245 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 464863 T2245 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 464863 T2245 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 464864 T2245 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 464864 T2245 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 464864 T2245 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 464864 T2245 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 464865 T2245 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 464865 T2245 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 464866 T2245 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 464868 T2245 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 464868 T2245 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 464868 T2245 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 464869 T2245 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@8a386f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@af05de),segFN=segments_1,generation=1}
   [junit4]   2> 464869 T2245 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 464869 T2245 oass.SolrIndexSearcher.<init> Opening Searcher@117ba9f[collection1] main
   [junit4]   2> 464869 T2245 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 464870 T2245 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 464870 T2245 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 464870 T2245 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 464871 T2245 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 464871 T2245 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 464871 T2245 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 464872 T2245 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 464872 T2245 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 464872 T2245 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 464874 T2246 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@117ba9f[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 464875 T2245 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 464876 T2249 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:48570 collection:collection1 shard:shard2
   [junit4]   2> 464876 T2200 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 464876 T2249 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 464876 T2200 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 464880 T2249 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 464881 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 464881 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 464881 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 464881 T2249 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 464882 T2249 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1891 name=collection1 org.apache.solr.core.SolrCore@1bb8544 url=http://127.0.0.1:48570/collection1 node=127.0.0.1:48570_ C1891_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:48570, core=collection1, node_name=127.0.0.1:48570_}
   [junit4]   2> 464882 T2249 C1891 P48570 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:48570/collection1/
   [junit4]   2> 464882 T2223 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 464882 T2249 C1891 P48570 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 464882 T2249 C1891 P48570 oasc.SyncStrategy.syncToMe http://127.0.0.1:48570/collection1/ has no replicas
   [junit4]   2> 464883 T2249 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:48570/collection1/ shard2
   [junit4]   2> 464883 T2249 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 464884 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 464891 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 464893 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 464996 T2231 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 464996 T2222 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 464996 T2244 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 465045 T2249 oasc.ZkController.register We are http://127.0.0.1:48570/collection1/ and leader is http://127.0.0.1:48570/collection1/
   [junit4]   2> 465045 T2249 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:48570
   [junit4]   2> 465045 T2249 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 465045 T2249 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 465046 T2249 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 465047 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 465047 T2249 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 465047 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 465047 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 465048 T2223 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 465049 T2223 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48570",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:48570_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 465051 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 465143 T2200 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 465144 T2200 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 465145 T2200 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:39553
   [junit4]   2> 465146 T2200 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 465146 T2200 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 465147 T2200 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-004
   [junit4]   2> 465147 T2200 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-004/'
   [junit4]   2> 465154 T2244 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 465154 T2222 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 465154 T2231 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 465158 T2200 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-004/solr.xml
   [junit4]   2> 465165 T2200 oasc.CoreContainer.<init> New CoreContainer 4232920
   [junit4]   2> 465165 T2200 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-004/]
   [junit4]   2> 465166 T2200 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 465166 T2200 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 465166 T2200 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 465167 T2200 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 465167 T2200 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 465167 T2200 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 465168 T2200 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 465168 T2200 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 465168 T2200 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 465169 T2200 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 465169 T2200 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 465169 T2200 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 465170 T2200 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:60165/solr
   [junit4]   2> 465170 T2200 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 465171 T2200 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 465172 T2260 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f28a9d name:ZooKeeperConnection Watcher:127.0.0.1:60165 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 465172 T2200 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 465173 T2200 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 465176 T2262 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@fe2c74 name:ZooKeeperConnection Watcher:127.0.0.1:60165/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 465176 T2200 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 465180 T2200 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 466182 T2200 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39553_
   [junit4]   2> 466184 T2200 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:39553_
   [junit4]   2> 466192 T2263 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 466192 T2263 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 466194 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 466194 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 466194 T2263 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 466194 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 466195 T2223 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 466195 T2223 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39553",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:39553_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 466196 T2223 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 466196 T2223 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 466197 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 466298 T2231 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 466299 T2222 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 466298 T2244 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 466299 T2262 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 467195 T2263 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 467195 T2263 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 467196 T2263 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 467196 T2263 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 467196 T2263 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 467197 T2263 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-004/collection1/'
   [junit4]   2> 467197 T2263 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 467197 T2263 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 467198 T2263 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 467215 T2263 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 467223 T2263 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 467224 T2263 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 467235 T2263 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 467317 T2263 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 467320 T2263 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 467322 T2263 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 467328 T2263 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 467330 T2263 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 467331 T2263 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 467341 T2263 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 467341 T2263 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 467342 T2263 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 467342 T2263 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 467343 T2263 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 467343 T2263 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 467343 T2263 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 467343 T2263 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty2/
   [junit4]   2> 467343 T2263 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ac36c0
   [junit4]   2> 467344 T2263 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty2
   [junit4]   2> 467344 T2263 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty2/index/
   [junit4]   2> 467344 T2263 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 467344 T2263 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty2/index
   [junit4]   2> 467345 T2263 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=18.86328125, floorSegmentMB=0.591796875, forceMergeDeletesPctAllowed=0.1200649045530755, segmentsPerTier=8.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2075689064913826
   [junit4]   2> 467345 T2263 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1914698 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1a2dbcc),segFN=segments_1,generation=1}
   [junit4]   2> 467345 T2263 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 467348 T2263 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 467348 T2263 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 467348 T2263 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 467348 T2263 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 467348 T2263 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 467349 T2263 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 467349 T2263 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 467349 T2263 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 467349 T2263 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 467350 T2263 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 467350 T2263 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 467350 T2263 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 467350 T2263 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 467351 T2263 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 467351 T2263 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 467351 T2263 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 467351 T2263 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 467353 T2263 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 467354 T2263 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 467354 T2263 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 467355 T2263 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 467355 T2263 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1914698 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1a2dbcc),segFN=segments_1,generation=1}
   [junit4]   2> 467355 T2263 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 467355 T2263 oass.SolrIndexSearcher.<init> Opening Searcher@10a63f6[collection1] main
   [junit4]   2> 467356 T2263 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 467356 T2263 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 467357 T2263 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 467357 T2263 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 467357 T2263 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 467357 T2263 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 467358 T2263 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 467358 T2263 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 467358 T2263 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 467358 T2263 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 467360 T2264 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@10a63f6[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 467362 T2263 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 467363 T2267 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:39553 collection:collection1 shard:shard1
   [junit4]   2> 467363 T2200 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 467364 T2200 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 467363 T2267 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 467367 T2267 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 467368 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 467368 T2267 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 467368 T2267 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 467368 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C1892 name=collection1 org.apache.solr.core.SolrCore@d49e1c url=http://127.0.0.1:39553/collection1 node=127.0.0.1:39553_ C1892_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:39553, core=collection1, node_name=127.0.0.1:39553_}
   [junit4]   2> 467368 T2267 C1892 P39553 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:39553/collection1/
   [junit4]   2> 467369 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 467369 T2267 C1892 P39553 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 467369 T2267 C1892 P39553 oasc.SyncStrategy.syncToMe http://127.0.0.1:39553/collection1/ has no replicas
   [junit4]   2> 467369 T2267 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:39553/collection1/ shard1
   [junit4]   2> 467370 T2267 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 467370 T2223 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 467372 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 467372 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 467375 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 467477 T2231 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 467477 T2262 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 467477 T2222 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 467477 T2244 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 467523 T2267 oasc.ZkController.register We are http://127.0.0.1:39553/collection1/ and leader is http://127.0.0.1:39553/collection1/
   [junit4]   2> 467523 T2267 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:39553
   [junit4]   2> 467524 T2267 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 467524 T2267 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 467524 T2267 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 467526 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 467526 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 467526 T2267 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 467526 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 467526 T2223 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 467527 T2223 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39553",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:39553_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 467528 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 467613 T2200 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 467613 T2200 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 467615 T2200 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:51679
   [junit4]   2> 467616 T2200 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 467616 T2200 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 467617 T2200 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-005
   [junit4]   2> 467617 T2200 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-005/'
   [junit4]   2> 467627 T2200 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-005/solr.xml
   [junit4]   2> 467630 T2231 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 467630 T2222 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 467630 T2262 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 467630 T2244 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 467635 T2200 oasc.CoreContainer.<init> New CoreContainer 21008042
   [junit4]   2> 467635 T2200 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-005/]
   [junit4]   2> 467636 T2200 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 467636 T2200 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 467637 T2200 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 467637 T2200 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 467637 T2200 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 467637 T2200 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 467638 T2200 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 467638 T2200 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 467638 T2200 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 467639 T2200 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 467639 T2200 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 467639 T2200 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 467640 T2200 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:60165/solr
   [junit4]   2> 467640 T2200 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 467641 T2200 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 467642 T2278 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@17e7976 name:ZooKeeperConnection Watcher:127.0.0.1:60165 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 467643 T2200 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 467644 T2200 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 467645 T2280 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@165beb2 name:ZooKeeperConnection Watcher:127.0.0.1:60165/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 467645 T2200 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 467650 T2200 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 468652 T2200 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51679_
   [junit4]   2> 468653 T2200 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51679_
   [junit4]   2> 468661 T2281 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 468661 T2281 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 468662 T2222 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4

[...truncated too long message...]

execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 481783 T2300 C1911 P59333 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
   [junit4]   2> 481783 T2300 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is shutdown
   [junit4]   2> 481784 T2300 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@cfbe4e
   [junit4]   2> 481793 T2300 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=7,autocommits=0,soft autocommits=1,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=1,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=42,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=6606,transaction_logs_total_number=6}
   [junit4]   2> 481793 T2300 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 481793 T2300 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 481794 T2300 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 481794 T2300 C1911 P59333 oasu.DirectUpdateHandler2.closeWriter Committing on IndexWriter close.
   [junit4]   2> 481794 T2300 C1911 P59333 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@8e405a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1a8a4d1),segFN=segments_7,generation=7}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@8e405a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1a8a4d1),segFN=segments_8,generation=8}
   [junit4]   2> 481795 T2300 C1911 P59333 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 8
   [junit4]   2> 481795 T2300 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 481795 T2300 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 481796 T2300 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty4/index [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty4/index;done=false>>]
   [junit4]   2> 481796 T2300 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty4/index
   [junit4]   2> 481796 T2300 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty4 [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty4;done=false>>]
   [junit4]   2> 481796 T2300 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001/tempDir-001/jetty4
   [junit4]   2> 481796 T2300 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 481797 T2300 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./temp/solr.cloud.BasicDistributedZkTest-B98D7890D2A4D556-001
   [junit4]   2> NOTE: test params are: codec=Lucene45, sim=DefaultSimilarity, locale=th_TH, timezone=Etc/GMT+8
   [junit4]   2> NOTE: Linux 3.8.0-41-generic i386/Oracle Corporation 1.8.0_20-ea (32-bit)/cpus=8,threads=1,free=16281824,total=228986880
   [junit4]   2> NOTE: All tests run in this JVM: [UniqFieldsUpdateProcessorFactoryTest, PingRequestHandlerTest, DOMUtilTest, TestFiltering, DirectUpdateHandlerTest, TestQueryTypes, DeleteReplicaTest, TestCustomSort, TestManagedSynonymFilterFactory, TestAddFieldRealTimeGet, TestReRankQParserPlugin, OpenExchangeRatesOrgProviderTest, PreAnalyzedUpdateProcessorTest, OutputWriterTest, BasicZkTest, NumericFieldsTest, TestIndexingPerformance, SolrXmlInZkTest, HighlighterTest, IndexBasedSpellCheckerTest, TestDefaultSearchFieldResource, DisMaxRequestHandlerTest, RequestHandlersTest, TestIBSimilarityFactory, BadCopyFieldTest, HdfsBasicDistributedZk2Test, SignatureUpdateProcessorFactoryTest, CursorMarkTest, OverseerRolesTest, SolrCmdDistributorTest, TestSchemaNameResource, TestQuerySenderNoQuery, SolrRequestParserTest, ConnectionManagerTest, TestLRUCache, TestDFRSimilarityFactory, TestCollapseQParserPlugin, TestXIncludeConfig, TestComponentsName, TestLazyCores, FieldAnalysisRequestHandlerTest, DistanceFunctionTest, ConvertedLegacyTest, HdfsCollectionsAPIDistributedZkTest, SolrIndexConfigTest, TestShardHandlerFactory, TestWordDelimiterFilterFactory, TestSolr4Spatial, PrimUtilsTest, TestJmxIntegration, HighlighterConfigTest, SuggesterWFSTTest, TestPartialUpdateDeduplication, NoCacheHeaderTest, TestSolrJ, TestUpdate, DocumentAnalysisRequestHandlerTest, DistributedSuggestComponentTest, TestClassNameShortening, SpellCheckCollatorTest, TestDocBasedVersionConstraints, TestZkChroot, AlternateDirectoryTest, TestPhraseSuggestions, ResponseHeaderTest, JSONWriterTest, SpellCheckComponentTest, BadIndexSchemaTest, TestAtomicUpdateErrorCases, XmlUpdateRequestHandlerTest, TestStressLucene, TestReload, TestRandomFaceting, DocValuesMultiTest, ShardRoutingCustomTest, TestInfoStreamLogging, CurrencyFieldOpenExchangeTest, TestImplicitCoreProperties, TestExpandComponent, CoreContainerCoreInitFailuresTest, CachingDirectoryFactoryTest, SSLMigrationTest, TestExceedMaxTermLength, TestNoOpRegenerator, TestLMDirichletSimilarityFactory, TestHashPartitioner, TestFileDictionaryLookup, TestFreeTextSuggestions, CoreMergeIndexesAdminHandlerTest, TestSolrCoreProperties, CurrencyFieldXmlFileTest, TestDistributedGrouping, TestFieldResource, TestCoreDiscovery, SolrPluginUtilsTest, ResourceLoaderTest, TestBinaryResponseWriter, TestSolrXmlPersistence, RemoteQueryErrorTest, DirectSolrConnectionTest, SimplePostToolTest, BasicDistributedZkTest]
   [junit4] Completed on J1 in 21.74s, 1 test, 1 failure <<< FAILURES!

[...truncated 727 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 410 suites, 1693 tests, 1 failure, 36 ignored (15 assumptions)

Total time: 69 minutes 51 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.8.0_20-ea-b15 -server -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.8.0_20-ea-b15) - Build # 10503 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/10503/
Java: 32bit/jdk1.8.0_20-ea-b15 -server -XX:+UseG1GC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.testDistribSearch

Error Message:
No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1
	at __randomizedtesting.SeedInfo.seed([6827B069CE5BDA0F:E9C13E71B904BA33]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:567)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.doTest(ChaosMonkeyNothingIsSafeTest.java:203)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:863)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11657 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/init-core-data-001
   [junit4]   2> 1265673 T4864 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1265679 T4864 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 1265679 T4864 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1265680 T4865 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1265780 T4864 oasc.ZkTestServer.run start zk server on port:35405
   [junit4]   2> 1265782 T4864 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1265913 T4871 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@13d0bd name:ZooKeeperConnection Watcher:127.0.0.1:35405 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1265913 T4864 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1265913 T4864 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1265918 T4864 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1265919 T4873 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b7fb6 name:ZooKeeperConnection Watcher:127.0.0.1:35405/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1265919 T4864 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1265920 T4864 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1265922 T4864 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1265923 T4864 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1265925 T4864 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1265926 T4864 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1265927 T4864 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1265929 T4864 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1265930 T4864 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1265933 T4864 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1265934 T4864 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1265936 T4864 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1265936 T4864 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1265938 T4864 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1265938 T4864 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1265940 T4864 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1265940 T4864 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1265942 T4864 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1265942 T4864 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1265944 T4864 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1265944 T4864 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1265946 T4864 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1265946 T4864 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1265948 T4864 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1265949 T4864 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1265951 T4864 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1265951 T4864 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1266215 T4864 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1266217 T4864 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:37118
   [junit4]   2> 1266218 T4864 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1266218 T4864 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1266219 T4864 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-002
   [junit4]   2> 1266219 T4864 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-002/'
   [junit4]   2> 1266234 T4864 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-002/solr.xml
   [junit4]   2> 1266243 T4864 oasc.CoreContainer.<init> New CoreContainer 25260626
   [junit4]   2> 1266244 T4864 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-002/]
   [junit4]   2> 1266245 T4864 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1266245 T4864 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1266245 T4864 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1266246 T4864 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1266246 T4864 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1266246 T4864 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1266247 T4864 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1266247 T4864 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1266248 T4864 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1266248 T4864 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1266249 T4864 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1266249 T4864 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1266250 T4864 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35405/solr
   [junit4]   2> 1266250 T4864 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1266251 T4864 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1266252 T4884 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f7feae name:ZooKeeperConnection Watcher:127.0.0.1:35405 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1266252 T4864 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1266254 T4864 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1266256 T4886 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15978f3 name:ZooKeeperConnection Watcher:127.0.0.1:35405/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1266256 T4864 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1266257 T4864 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1266260 T4864 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1266265 T4864 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1266268 T4864 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1266270 T4864 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1266272 T4864 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1266273 T4864 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37118_
   [junit4]   2> 1266274 T4864 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37118_
   [junit4]   2> 1266275 T4864 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1266277 T4864 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1266279 T4864 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:37118_
   [junit4]   2> 1266279 T4864 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1266281 T4864 oasc.Overseer.start Overseer (id=91894805947285507-127.0.0.1:37118_-n_0000000000) starting
   [junit4]   2> 1266283 T4864 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1266289 T4888 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1266289 T4864 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1266292 T4864 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1266294 T4864 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1266297 T4887 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1266300 T4889 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1266301 T4889 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1266302 T4889 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1266303 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1266304 T4887 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1266304 T4887 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37118",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:37118_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1266304 T4887 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 1266305 T4887 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1266306 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1266306 T4886 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1267302 T4889 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1267303 T4889 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1267304 T4889 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1267304 T4889 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1267304 T4889 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1267305 T4889 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-002/collection1/'
   [junit4]   2> 1267305 T4889 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 1267306 T4889 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 1267306 T4889 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 1267329 T4889 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 1267339 T4889 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1267340 T4889 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1267346 T4889 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1267460 T4889 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1267462 T4889 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1267463 T4889 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1267465 T4889 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1267479 T4889 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1267479 T4889 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 1267480 T4889 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/control/data/
   [junit4]   2> 1267480 T4889 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@16a0e2c
   [junit4]   2> 1267480 T4889 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/control/data
   [junit4]   2> 1267481 T4889 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/control/data/index/
   [junit4]   2> 1267481 T4889 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1267482 T4889 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/control/data/index
   [junit4]   2> 1267482 T4889 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=91.4892578125, floorSegmentMB=0.31640625, forceMergeDeletesPctAllowed=17.7618452998192, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.16537432853169493
   [junit4]   2> 1267487 T4889 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1267487 T4889 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1267492 T4889 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1267493 T4889 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1267493 T4889 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1267493 T4889 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1267493 T4889 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1267494 T4889 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1267494 T4889 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1267494 T4889 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1267494 T4889 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1267495 T4889 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1267495 T4889 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1267495 T4889 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1267496 T4889 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1267496 T4889 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1267496 T4889 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1267497 T4889 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1267497 T4889 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1267499 T4889 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1267500 T4889 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1267500 T4889 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1267501 T4889 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=19.7138671875, floorSegmentMB=1.419921875, forceMergeDeletesPctAllowed=10.121384593559947, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.23167308668280784
   [junit4]   2> 1267502 T4889 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1267502 T4889 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1267502 T4889 oass.SolrIndexSearcher.<init> Opening Searcher@185eddd[collection1] main
   [junit4]   2> 1267502 T4889 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1267503 T4889 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1267503 T4889 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1267504 T4889 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1267504 T4889 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1267504 T4889 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1267505 T4889 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1267505 T4889 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1267505 T4889 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1267506 T4889 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1267507 T4890 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@185eddd[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 1267508 T4889 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1267509 T4893 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:37118 collection:control_collection shard:shard1
   [junit4]   2> 1267509 T4864 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 1267510 T4864 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1267510 T4893 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1267511 T4864 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1267513 T4895 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d2b6ec name:ZooKeeperConnection Watcher:127.0.0.1:35405/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1267513 T4864 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1267514 T4864 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1267515 T4893 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1267517 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1267517 T4893 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1267517 T4864 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1267518 T4887 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1267517 T4893 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1492 name=collection1 org.apache.solr.core.SolrCore@128936d url=http://127.0.0.1:37118/collection1 node=127.0.0.1:37118_ C1492_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:37118, core=collection1, node_name=127.0.0.1:37118_}
   [junit4]   2> 1267518 T4893 C1492 P37118 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:37118/collection1/
   [junit4]   2> 1267519 T4893 C1492 P37118 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1267519 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1267519 T4893 C1492 P37118 oasc.SyncStrategy.syncToMe http://127.0.0.1:37118/collection1/ has no replicas
   [junit4]   2> 1267520 T4893 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:37118/collection1/ shard1
   [junit4]   2> 1267520 T4893 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1267525 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1267527 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1267628 T4886 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1267628 T4895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1267676 T4893 oasc.ZkController.register We are http://127.0.0.1:37118/collection1/ and leader is http://127.0.0.1:37118/collection1/
   [junit4]   2> 1267676 T4893 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:37118
   [junit4]   2> 1267677 T4893 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1267677 T4893 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1267677 T4893 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1267678 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1267679 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1267679 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1267680 T4893 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1267681 T4887 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1267681 T4887 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37118",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:37118_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1267683 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1267786 T4895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1267786 T4886 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1267890 T4864 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1267891 T4864 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1267892 T4864 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:36960
   [junit4]   2> 1267893 T4864 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1267893 T4864 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1267894 T4864 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-003
   [junit4]   2> 1267894 T4864 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-003/'
   [junit4]   2> 1267911 T4864 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-003/solr.xml
   [junit4]   2> 1267919 T4864 oasc.CoreContainer.<init> New CoreContainer 6191314
   [junit4]   2> 1267920 T4864 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-003/]
   [junit4]   2> 1267921 T4864 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1267921 T4864 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1267922 T4864 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1267922 T4864 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1267922 T4864 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1267923 T4864 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1267923 T4864 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1267924 T4864 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1267924 T4864 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1267925 T4864 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1267925 T4864 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1267926 T4864 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1267926 T4864 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35405/solr
   [junit4]   2> 1267927 T4864 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1267929 T4864 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1267930 T4906 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b67dee name:ZooKeeperConnection Watcher:127.0.0.1:35405 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1267930 T4864 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1267932 T4864 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1267933 T4908 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e1e6c3 name:ZooKeeperConnection Watcher:127.0.0.1:35405/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1267933 T4864 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1267938 T4864 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1268940 T4864 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36960_
   [junit4]   2> 1268943 T4864 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:36960_
   [junit4]   2> 1268949 T4909 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1268949 T4909 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1268950 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1268950 T4909 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1268950 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1268951 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1268951 T4887 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1268952 T4887 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36960",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:36960_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1268952 T4887 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 1268952 T4887 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1268954 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1269055 T4908 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1269055 T4886 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1269055 T4895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1269950 T4909 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1269951 T4909 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1269951 T4909 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1269952 T4909 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1269953 T4909 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1269953 T4909 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-003/collection1/'
   [junit4]   2> 1269954 T4909 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 1269954 T4909 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 1269954 T4909 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 1269980 T4909 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 1269990 T4909 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1269991 T4909 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1269998 T4909 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1270128 T4909 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1270130 T4909 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1270131 T4909 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1270133 T4909 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1270147 T4909 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1270148 T4909 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 1270148 T4909 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty1/
   [junit4]   2> 1270148 T4909 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@16a0e2c
   [junit4]   2> 1270149 T4909 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty1
   [junit4]   2> 1270149 T4909 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty1/index/
   [junit4]   2> 1270150 T4909 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1270150 T4909 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty1/index
   [junit4]   2> 1270151 T4909 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=91.4892578125, floorSegmentMB=0.31640625, forceMergeDeletesPctAllowed=17.7618452998192, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.16537432853169493
   [junit4]   2> 1270157 T4909 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 1270157 T4909 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1270161 T4909 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1270161 T4909 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1270162 T4909 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1270162 T4909 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1270162 T4909 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1270162 T4909 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1270163 T4909 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1270163 T4909 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1270163 T4909 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1270164 T4909 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1270164 T4909 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1270165 T4909 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1270165 T4909 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1270165 T4909 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1270166 T4909 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1270166 T4909 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1270167 T4909 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1270169 T4909 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1270170 T4909 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1270170 T4909 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1270171 T4909 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=19.7138671875, floorSegmentMB=1.419921875, forceMergeDeletesPctAllowed=10.121384593559947, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.23167308668280784
   [junit4]   2> 1270172 T4909 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 1270172 T4909 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1270173 T4909 oass.SolrIndexSearcher.<init> Opening Searcher@a4ace8[collection1] main
   [junit4]   2> 1270173 T4909 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1270174 T4909 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1270174 T4909 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1270175 T4909 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1270175 T4909 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1270175 T4909 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1270176 T4909 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1270176 T4909 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1270176 T4909 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1270176 T4909 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1270179 T4910 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@a4ace8[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 1270181 T4909 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1270182 T4913 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:36960 collection:collection1 shard:shard2
   [junit4]   2> 1270183 T4864 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 1270183 T4864 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1270183 T4913 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1270188 T4913 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1270190 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1270190 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1270190 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1270193 T4913 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1270193 T4913 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1270193 T4887 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C1493 name=collection1 org.apache.solr.core.SolrCore@a4db0b url=http://127.0.0.1:36960/collection1 node=127.0.0.1:36960_ C1493_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:36960, core=collection1, node_name=127.0.0.1:36960_}
   [junit4]   2> 1270193 T4913 C1493 P36960 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:36960/collection1/
   [junit4]   2> 1270195 T4913 C1493 P36960 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1270195 T4913 C1493 P36960 oasc.SyncStrategy.syncToMe http://127.0.0.1:36960/collection1/ has no replicas
   [junit4]   2> 1270196 T4913 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:36960/collection1/ shard2
   [junit4]   2> 1270196 T4913 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1270198 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1270202 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1270204 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1270306 T4886 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1270306 T4895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1270306 T4908 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1270354 T4913 oasc.ZkController.register We are http://127.0.0.1:36960/collection1/ and leader is http://127.0.0.1:36960/collection1/
   [junit4]   2> 1270354 T4913 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:36960
   [junit4]   2> 1270354 T4913 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1270354 T4913 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1270355 T4913 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1270356 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1270356 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1270356 T4913 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1270356 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1270357 T4887 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1270358 T4887 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36960",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:36960_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1270360 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1270463 T4908 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1270463 T4886 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1270464 T4895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1270533 T4864 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 1270534 T4864 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1270536 T4864 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:38168
   [junit4]   2> 1270537 T4864 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1270537 T4864 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1270538 T4864 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-004
   [junit4]   2> 1270538 T4864 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-004/'
   [junit4]   2> 1270554 T4864 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-004/solr.xml
   [junit4]   2> 1270563 T4864 oasc.CoreContainer.<init> New CoreContainer 30900302
   [junit4]   2> 1270563 T4864 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-004/]
   [junit4]   2> 1270564 T4864 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1270565 T4864 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1270565 T4864 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1270566 T4864 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1270566 T4864 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1270566 T4864 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1270567 T4864 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1270567 T4864 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1270567 T4864 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1270568 T4864 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1270569 T4864 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1270569 T4864 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1270569 T4864 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35405/solr
   [junit4]   2> 1270570 T4864 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1270571 T4864 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1270573 T4924 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@103fbdc name:ZooKeeperConnection Watcher:127.0.0.1:35405 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1270574 T4864 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1270576 T4864 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1270577 T4926 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@df1014 name:ZooKeeperConnection Watcher:127.0.0.1:35405/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1270578 T4864 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1270593 T4864 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1271596 T4864 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38168_
   [junit4]   2> 1271598 T4864 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38168_
   [junit4]   2> 1271608 T4927 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1271609 T4927 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1271610 T4927 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1271610 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1271611 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1271611 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1271612 T4887 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1271613 T4887 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38168",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38168_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1271614 T4887 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 1271614 T4887 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1271617 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1271720 T4908 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1271720 T4886 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1271720 T4926 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1271720 T4895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1272611 T4927 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1272611 T4927 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1272612 T4927 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1272612 T4927 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1272614 T4927 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1272614 T4927 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-004/collection1/'
   [junit4]   2> 1272615 T4927 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 1272615 T4927 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 1272616 T4927 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 1272641 T4927 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_5_0
   [junit4]   2> 1272650 T4927 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1272652 T4927 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1272658 T4927 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1272755 T4927 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1272757 T4927 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1272758 T4927 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1272761 T4927 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1272773 T4927 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1272774 T4927 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 1272774 T4927 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty2/
   [junit4]   2> 1272774 T4927 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@16a0e2c
   [junit4]   2> 1272775 T4927 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty2
   [junit4]   2> 1272775 T4927 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty2/index/
   [junit4]   2> 1272775 T4927 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1272776 T4927 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty2/index
   [junit4]   2> 1272776 T4927 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=91.4892578125, floorSegmentMB=0.31640625, forceMergeDeletesPctAllowed=17.7618452998192, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.16537432853169493
   [junit4]   2> 1272782 T4927 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty2/index,segFN=segments_1,generation=1}
   [junit4]   2> 1272782 T4927 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1272787 T4927 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1272788 T4927 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1272788 T4927 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1272788 T4927 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1272789 T4927 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1272789 T4927 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1272790 T4927 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1272790 T4927 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1272790 T4927 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1272791 T4927 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1272792 T4927 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1272792 T4927 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1272793 T4927 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1272793 T4927 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1272794 T4927 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1272795 T4927 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1272795 T4927 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1272798 T4927 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1272800 T4927 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1272801 T4927 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1272802 T4927 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=19.7138671875, floorSegmentMB=1.419921875, forceMergeDeletesPctAllowed=10.121384593559947, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.23167308668280784
   [junit4]   2> 1272803 T4927 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty2/index,segFN=segments_1,generation=1}
   [junit4]   2> 1272803 T4927 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1272804 T4927 oass.SolrIndexSearcher.<init> Opening Searcher@a069b[collection1] main
   [junit4]   2> 1272804 T4927 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1272805 T4927 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1272806 T4927 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1272806 T4927 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1272806 T4927 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1272806 T4927 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1272807 T4927 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1272807 T4927 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1272807 T4927 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1272808 T4927 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1272810 T4928 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@a069b[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 1272811 T4927 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1272812 T4931 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:38168 collection:collection1 shard:shard1
   [junit4]   2> 1272812 T4864 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 1272813 T4864 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1272813 T4931 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1272817 T4931 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1272819 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1272819 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1272819 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1272820 T4931 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1272820 T4931 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1272820 T4887 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C1494 name=collection1 org.apache.solr.core.SolrCore@125901a url=http://127.0.0.1:38168/collection1 node=127.0.0.1:38168_ C1494_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:38168, core=collection1, node_name=127.0.0.1:38168_}
   [junit4]   2> 1272820 T4931 C1494 P38168 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:38168/collection1/
   [junit4]   2> 1272821 T4931 C1494 P38168 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1272821 T4931 C1494 P38168 oasc.SyncStrategy.syncToMe http://127.0.0.1:38168/collection1/ has no replicas
   [junit4]   2> 1272821 T4931 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:38168/collection1/ shard1
   [junit4]   2> 1272821 T4931 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1272824 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1272825 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1272827 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1272928 T4908 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1272928 T4886 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1272928 T4895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1272928 T4926 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1272976 T4931 oasc.ZkController.register We are http://127.0.0.1:38168/collection1/ and leader is http://127.0.0.1:38168/collection1/
   [junit4]   2> 1272976 T4931 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:38168
   [junit4]   2> 1272977 T4931 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1272977 T4931 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1272977 T4931 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1272978 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1272978 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1272978 T4931 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1272978 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1272980 T4887 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1272980 T4887 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38168",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38168_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 1272982 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1273072 T4864 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 1273073 T4864 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1273075 T4864 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:41350
   [junit4]   2> 1273076 T4864 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1273077 T4864 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1273077 T4864 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-005
   [junit4]   2> 1273078 T4864 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-005/'
   [junit4]   2> 1273087 T4926 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1273087 T4886 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1273087 T4895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1273087 T4908 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1273094 T4864 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-005/solr.xml
   [junit4]   2> 1273102 T4864 oasc.CoreContainer.<init> New CoreContainer 20186768
   [junit4]   2> 1273102 T4864 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-005/]
   [junit4]   2> 1273103 T4864 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1273103 T4864 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1273104 T4864 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1273104 T4864 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1273105 T4864 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1273105 T4864 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1273105 T4864 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1273106 T4864 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1273106 T4864 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1273107 T4864 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1273107 T4864 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1273108 T4864 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1273108 T4864 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35405/solr
   [junit4]   2> 1273109 T4864 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1273110 T4864 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1273111 T4942 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1911cf0 name:ZooKeeperConnection Watcher:127.0.0.1:35405 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1273111 T4864 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1273113 T4864 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1273115 T4944 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c9499d name:ZooKeeperConnection Watcher:127.0.0.1:35405/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1273115 T4864 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1273121 T4864 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1274124 T4864 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41350_
   [junit4]   2> 1274127 T4864 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41350_
   [junit4]   2> 1274132 T4945 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1274133 T4945 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1274133 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1274133 T4945 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1274133 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1274134 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1274134 T4887 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1274135 T4887 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41350",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:41350_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1274135 T4887 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 1274135 T4887 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1274137 T4886 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildre

[...truncated too long message...]

0.0.1:56801, core=collection1, node_name=127.0.0.1:56801_}
   [junit4]   2> 1340052 T5026 C1517 P56801 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
   [junit4]   2> 1340053 T5026 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is shutdown
   [junit4]   2> 1340053 T5026 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@180c85f
   [junit4]   2> 1340059 T5026 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=47,cumulative_deletesById=23,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=5430,transaction_logs_total_number=1}
   [junit4]   2> 1340059 T5026 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1340059 T5026 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1340059 T5026 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1340061 T5026 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1340061 T5026 oasc.CachingDirectoryFactory.close Closing StandardDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1340061 T5026 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty7/index [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty7/index;done=false>>]
   [junit4]   2> 1340061 T5026 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty7/index
   [junit4]   2> 1340062 T5026 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty7 [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty7;done=false>>]
   [junit4]   2> 1340062 T5026 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001/tempDir-001/jetty7
   [junit4]   2> 1340062 T5026 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 1340062 T5026 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./temp/solr.cloud.ChaosMonkeyNothingIsSafeTest-6827B069CE5BDA0F-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=mt_MT, timezone=Pacific/Palau
   [junit4]   2> NOTE: Linux 3.8.0-41-generic i386/Oracle Corporation 1.8.0_20-ea (32-bit)/cpus=8,threads=1,free=325091080,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [AliasIntegrationTest, AnalysisAfterCoreReloadTest, ExternalFileFieldSortTest, CopyFieldTest, TestWriterPerf, TestDocSet, TestAnalyzeInfixSuggestions, SliceStateUpdateTest, UpdateRequestProcessorFactoryTest, ModifyConfFileTest, RAMDirectoryFactoryTest, CoreAdminHandlerTest, TestSolrDeletionPolicy1, SolrIndexSplitterTest, TestFastLRUCache, AddSchemaFieldsUpdateProcessorFactoryTest, TestConfig, RegexBoostProcessorTest, DateRangeFieldTest, SOLR749Test, TestConfigSets, QueryElevationComponentTest, TestComplexPhraseQParserPlugin, SoftAutoCommitTest, TestRealTimeGet, TestSuggestSpellingConverter, RangeFacetTest, StatelessScriptUpdateProcessorFactoryTest, ScriptEngineTest, MoreLikeThisHandlerTest, ReturnFieldsTest, TestRemoteStreaming, TestBlendedInfixSuggestions, ResponseLogComponentTest, ExpressionTest, TestNRTOpen, TestManagedSchemaFieldResource, StressHdfsTest, TestCollectionAPI, HdfsUnloadDistributedZkTest, TestCursorMarkWithoutUniqueKey, BlockCacheTest, SolrTestCaseJ4Test, DistributedTermsComponentTest, URLClassifyProcessorTest, TestSearchPerf, LukeRequestHandlerTest, DocValuesMissingTest, TestSchemaVersionResource, CacheHeaderTest, TestOrdValues, CSVRequestHandlerTest, AddBlockUpdateTest, ChangedSchemaMergeTest, HdfsSyncSliceTest, PreAnalyzedFieldTest, SpatialFilterTest, TestLFUCache, DocValuesTest, QueryResultKeyTest, HttpPartitionTest, ShardSplitTest, TestStressVersions, SuggesterFSTTest, SampleTest, MergeStrategyTest, TestSearcherReuse, TestJmxMonitoredMap, TestGroupingSearch, TestSolrQueryParser, TestFunctionQuery, TestModifyConfFiles, TestTrie, SearchHandlerTest, BasicFunctionalityTest, BinaryUpdateRequestHandlerTest, TestReversedWildcardFilterFactory, SolrCoreTest, TestBM25SimilarityFactory, TestFaceting, CircularListTest, CoreAdminCreateDiscoverTest, SuggesterTest, TestLuceneMatchVersion, TestSystemIdResolver, TestMergePolicyConfig, TestCharFilters, TestSort, TestStandardQParsers, ClusterStateTest, TestSimpleQParserPlugin, AbstractAnalyticsFacetTest, DistribCursorPagingTest, TestPHPSerializedResponseWriter, ZkCLITest, TestRandomDVFaceting, SliceStateTest, HdfsLockFactoryTest, BasicDistributedZkTest, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, BasicZkTest, TestReplicationHandler, ClusterStateUpdateTest, LeaderElectionTest, TestZkChroot, AutoCommitTest, TestHashPartitioner, TestJoin, SimpleFacetsTest, PeerSyncTest, TestLazyCores, TestBadConfig, TestIndexSearcher, CurrencyFieldOpenExchangeTest, SimplePostToolTest, TestCoreDiscovery, SignatureUpdateProcessorFactoryTest, SuggesterTSTTest, PolyFieldTest, SchemaVersionSpecificBehaviorTest, TestPseudoReturnFields, TestWordDelimiterFilterFactory, DirectUpdateHandlerOptimizeTest, SortByFunctionTest, XsltUpdateRequestHandlerTest, IndexBasedSpellCheckerTest, TestQueryUtils, DirectSolrSpellCheckerTest, TestOmitPositions, TermsComponentTest, DocumentBuilderTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, FastVectorHighlighterTest, LoggingHandlerTest, IndexSchemaTest, TestCollationField, MBeansHandlerTest, QueryParsingTest, JsonLoaderTest, TestComponentsName, TestBinaryResponseWriter, UpdateParamsTest, BadComponentTest, MultiTermTest, NumericFieldsTest, MinimalSchemaTest, TestElisionMultitermQuery, TestPostingsSolrHighlighter, NotRequiredUniqueKeyTest, SpellPossibilityIteratorTest, TestCodecSupport, SynonymTokenizerTest, TestXIncludeConfig, EchoParamsTest, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestDefaultSimilarityFactory, TestFastWriter, TimeZoneUtilsTest, ResourceLoaderTest, TestFastOutputStream, PluginInfoTest, ChaosMonkeyNothingIsSafeTest]
   [junit4] Completed on J0 in 75.02s, 1 test, 1 error <<< FAILURES!

[...truncated 213 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1296: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:920: There were test failures: 410 suites, 1693 tests, 1 error, 36 ignored (15 assumptions)

Total time: 58 minutes 33 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.8.0_20-ea-b15 -server -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any