You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/09/15 21:49:02 UTC

[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.8.0) - Build # 1833 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/1833/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([9D91BDE760C23FDB:1C7733FF179D5FE7]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.requestWithRetryOnStaleState(CloudSolrServer.java:706)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:601)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrServer.deleteByQuery(SolrServer.java:285)
	at org.apache.solr.client.solrj.SolrServer.deleteByQuery(SolrServer.java:271)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:728)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.tryDelete(ChaosMonkeySafeLeaderTest.java:194)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:112)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
	at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11288 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/init-core-data-001
   [junit4]   2> 2658301 T7128 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 2658302 T7128 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 2658309 T7128 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 2658313 T7128 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2658315 T7129 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 2658415 T7128 oasc.ZkTestServer.run start zk server on port:51023
   [junit4]   2> 2658416 T7128 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2658422 T7128 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2658430 T7135 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@405ed72e name:ZooKeeperConnection Watcher:127.0.0.1:51023 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2658431 T7128 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2658431 T7128 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2658433 T7128 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 2658444 T7128 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2658448 T7128 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2658451 T7137 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6d90284e name:ZooKeeperConnection Watcher:127.0.0.1:51023/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2658451 T7128 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2658452 T7128 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2658452 T7128 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 2658457 T7128 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 2658464 T7128 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 2658467 T7128 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 2658473 T7128 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2658473 T7128 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 2658481 T7128 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 2658482 T7128 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 2658489 T7128 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2658489 T7128 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2658497 T7128 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2658498 T7128 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 2658506 T7128 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2658507 T7128 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 2658512 T7128 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2658512 T7128 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 2658520 T7128 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2658520 T7128 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 2658528 T7128 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2658529 T7128 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2658536 T7128 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2658536 T7128 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2658558 T7128 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2658559 T7128 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 2658564 T7128 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2658568 T7128 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 2658576 T7128 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2658579 T7128 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2658580 T7139 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21519232 name:ZooKeeperConnection Watcher:127.0.0.1:51023/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2658580 T7128 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2658580 T7128 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2660459 T7128 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2660471 T7128 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2660473 T7128 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:51027
   [junit4]   2> 2660476 T7128 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2660478 T7128 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2660479 T7128 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-002
   [junit4]   2> 2660479 T7128 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-002/'
   [junit4]   2> 2660519 T7128 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-002/solr.xml
   [junit4]   2> 2660537 T7128 oasc.CoreContainer.<init> New CoreContainer 668295376
   [junit4]   2> 2660538 T7128 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-002/]
   [junit4]   2> 2660539 T7128 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2660539 T7128 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2660539 T7128 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2660540 T7128 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2660540 T7128 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2660540 T7128 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2660540 T7128 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2660541 T7128 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2660541 T7128 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2660543 T7128 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 2660545 T7128 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2660546 T7128 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2660546 T7128 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2660546 T7128 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51023/solr
   [junit4]   2> 2660547 T7128 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2660548 T7128 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2660550 T7128 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2660552 T7150 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7abb9609 name:ZooKeeperConnection Watcher:127.0.0.1:51023 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2660554 T7128 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2660555 T7128 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2660559 T7128 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2660560 T7152 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5d2e38f7 name:ZooKeeperConnection Watcher:127.0.0.1:51023/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2660560 T7128 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2660563 T7128 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 2660568 T7128 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 2660573 T7128 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 2660578 T7128 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 2660583 T7128 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 2660590 T7128 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 2660595 T7128 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51027_
   [junit4]   2> 2660597 T7128 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51027_
   [junit4]   2> 2660606 T7128 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 2660610 T7128 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 2660615 T7128 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 2660620 T7128 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:51027_
   [junit4]   2> 2660620 T7128 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2660627 T7128 oasc.Overseer.start Overseer (id=92458663216218116-127.0.0.1:51027_-n_0000000000) starting
   [junit4]   2> 2660635 T7128 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 2660665 T7128 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 2660666 T7154 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2660669 T7128 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 2660675 T7128 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 2660679 T7128 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2660693 T7153 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2660700 T7156 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 2660701 T7156 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2660703 T7157 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2660707 T7156 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2660711 T7153 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:51027",
   [junit4]   2> 	  "node_name":"127.0.0.1:51027_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 2660711 T7153 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 2660711 T7153 oasc.Overseer$ClusterStateUpdater.createCollection state version control_collection 1
   [junit4]   2> 2660711 T7153 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2660715 T7157 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2661708 T7156 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2661709 T7156 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 2661712 T7156 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2661712 T7156 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 2661718 T7156 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2661718 T7156 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-002/collection1/'
   [junit4]   2> 2661720 T7156 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 2661722 T7156 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 2661723 T7156 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 2661770 T7156 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 2661794 T7156 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2661796 T7156 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2661804 T7156 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2661915 T7156 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2661916 T7156 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2661917 T7156 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2661926 T7156 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2661990 T7156 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 2661991 T7156 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 2661991 T7156 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-002/collection1/, dataDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/control/data/
   [junit4]   2> 2661991 T7156 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@65527925
   [junit4]   2> 2661994 T7156 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/control/data
   [junit4]   2> 2661994 T7156 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/control/data/index/
   [junit4]   2> 2661995 T7156 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2661996 T7156 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/control/data/index
   [junit4]   2> 2661997 T7156 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=87.1201171875, floorSegmentMB=1.4130859375, forceMergeDeletesPctAllowed=11.502438378813286, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.13592693577772286
   [junit4]   2> 2662001 T7156 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 2662002 T7156 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2662006 T7156 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2662006 T7156 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2662006 T7156 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2662007 T7156 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2662007 T7156 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2662008 T7156 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2662008 T7156 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2662009 T7156 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2662009 T7156 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2662009 T7156 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2662010 T7156 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2662011 T7156 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2662012 T7156 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2662012 T7156 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2662013 T7156 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2662013 T7156 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2662014 T7156 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2662014 T7156 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2662015 T7156 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 2662015 T7156 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 2662023 T7156 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2662025 T7156 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2662027 T7156 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2662029 T7156 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2662042 T7156 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2662042 T7156 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2662043 T7156 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=22, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 2662045 T7156 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 2662045 T7156 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2662045 T7156 oass.SolrIndexSearcher.<init> Opening Searcher@200b6d83[collection1] main
   [junit4]   2> 2662046 T7156 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 2662050 T7156 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2662050 T7156 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 2662051 T7156 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2662052 T7156 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 2662052 T7156 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2662053 T7156 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 2662054 T7156 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2662055 T7156 oascc.SolrZkClient.makePath makePath: /configs/conf1/_rest_managed.json
   [junit4]   2> 2662060 T7156 oasr.ManagedResourceStorage$ZooKeeperStorageIO$1.close Wrote 38 bytes to new znode /configs/conf1/_rest_managed.json
   [junit4]   2> 2662061 T7156 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2662061 T7156 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 2662064 T7158 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@200b6d83[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 2662066 T7156 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2662069 T7161 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:51027 collection:control_collection shard:shard1
   [junit4]   2> 2662070 T7128 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 2662070 T7128 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2662071 T7161 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 2662074 T7128 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2662075 T7128 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2662080 T7163 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f5f991c name:ZooKeeperConnection Watcher:127.0.0.1:51023/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2662080 T7128 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2662081 T7128 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2662084 T7128 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2662088 T7161 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2662094 T7128 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2662101 T7157 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2662105 T7161 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2662106 T7161 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C4938 name=collection1 org.apache.solr.core.SolrCore@166ee827 url=https://127.0.0.1:51027/collection1 node=127.0.0.1:51027_ C4938_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=https://127.0.0.1:51027, node_name=127.0.0.1:51027_, state=down}
   [junit4]   2> 2662106 T7161 C4938 P51027 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:51027/collection1/
   [junit4]   2> 2662107 T7161 C4938 P51027 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2662107 T7161 C4938 P51027 oasc.SyncStrategy.syncToMe https://127.0.0.1:51027/collection1/ has no replicas
   [junit4]   2> 2662107 T7161 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:51027/collection1/ shard1
   [junit4]   2> 2662107 T7161 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 2662161 T7157 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2662266 T7157 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2662267 T7164 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2662303 T7161 oasc.ZkController.register We are https://127.0.0.1:51027/collection1/ and leader is https://127.0.0.1:51027/collection1/
   [junit4]   2> 2662303 T7161 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:51027
   [junit4]   2> 2662303 T7161 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2662304 T7161 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 2662304 T7161 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2662311 T7157 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2662326 T7153 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:51027",
   [junit4]   2> 	  "node_name":"127.0.0.1:51027_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 2662434 T7164 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2662434 T7157 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2663891 T7128 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 2663892 T7128 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2663901 T7128 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2663903 T7128 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:51031
   [junit4]   2> 2663909 T7128 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2663909 T7128 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2663910 T7128 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-003
   [junit4]   2> 2663910 T7128 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-003/'
   [junit4]   2> 2663946 T7128 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-003/solr.xml
   [junit4]   2> 2663963 T7128 oasc.CoreContainer.<init> New CoreContainer 1316883938
   [junit4]   2> 2663964 T7128 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-003/]
   [junit4]   2> 2663965 T7128 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2663965 T7128 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2663966 T7128 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2663966 T7128 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2663967 T7128 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2663967 T7128 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2663967 T7128 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2663968 T7128 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2663968 T7128 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2663970 T7128 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 2663972 T7128 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2663972 T7128 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2663974 T7128 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2663974 T7128 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51023/solr
   [junit4]   2> 2663974 T7128 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2663974 T7128 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2663976 T7128 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2663981 T7175 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@32445441 name:ZooKeeperConnection Watcher:127.0.0.1:51023 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2663982 T7128 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2663983 T7128 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2663985 T7128 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2663988 T7177 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7de157ab name:ZooKeeperConnection Watcher:127.0.0.1:51023/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2663989 T7128 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2664008 T7128 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2665025 T7128 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51031_
   [junit4]   2> 2665033 T7128 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51031_
   [junit4]   2> 2665049 T7128 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 2665061 T7179 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 2665062 T7179 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2665064 T7157 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2665064 T7179 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2665076 T7153 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:51031",
   [junit4]   2> 	  "node_name":"127.0.0.1:51031_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 2665077 T7153 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
   [junit4]   2> 2665077 T7153 oasc.Overseer$ClusterStateUpdater.createCollection state version collection1 1
   [junit4]   2> 2665077 T7153 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2665085 T7164 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2665086 T7178 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2665086 T7157 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2666066 T7179 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2666067 T7179 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 2666071 T7179 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2666072 T7179 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2666075 T7179 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2666076 T7179 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-003/collection1/'
   [junit4]   2> 2666078 T7179 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 2666079 T7179 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 2666080 T7179 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 2666138 T7179 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 2666163 T7179 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2666166 T7179 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2666170 T7179 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2666296 T7179 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2666297 T7179 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2666298 T7179 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2666304 T7179 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2666381 T7179 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 2666382 T7179 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 2666383 T7179 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-003/collection1/, dataDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/jetty1/
   [junit4]   2> 2666383 T7179 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@65527925
   [junit4]   2> 2666384 T7179 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/jetty1
   [junit4]   2> 2666386 T7179 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/jetty1/index/
   [junit4]   2> 2666386 T7179 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 2666388 T7179 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/jetty1/index
   [junit4]   2> 2666389 T7179 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=87.1201171875, floorSegmentMB=1.4130859375, forceMergeDeletesPctAllowed=11.502438378813286, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.13592693577772286
   [junit4]   2> 2666393 T7179 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 2666393 T7179 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2666399 T7179 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2666400 T7179 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2666400 T7179 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2666400 T7179 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2666403 T7179 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2666403 T7179 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2666404 T7179 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2666404 T7179 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2666404 T7179 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2666405 T7179 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2666406 T7179 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2666406 T7179 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2666407 T7179 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2666408 T7179 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2666408 T7179 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2666409 T7179 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2666409 T7179 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2666410 T7179 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2666410 T7179 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 2666411 T7179 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 2666417 T7179 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2666418 T7179 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2666419 T7179 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2666421 T7179 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2666424 T7179 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2666424 T7179 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2666425 T7179 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=22, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 2666426 T7179 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 2666426 T7179 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2666427 T7179 oass.SolrIndexSearcher.<init> Opening Searcher@570a0ed1[collection1] main
   [junit4]   2> 2666427 T7179 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2666431 T7179 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2666431 T7179 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 2666433 T7179 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2666434 T7179 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 2666434 T7179 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2666438 T7179 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream Read 38 bytes from znode /configs/conf1/_rest_managed.json
   [junit4]   2> 2666438 T7179 oasr.ManagedResourceStorage.load Loaded LinkedHashMap at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2666439 T7179 oasr.ManagedResource.reloadFromStorage Loaded initArgs {} for /rest/managed
   [junit4]   2> 2666439 T7179 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 2666446 T7180 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@570a0ed1[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 2666449 T7179 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2666453 T7183 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:51031 collection:collection1 shard:shard1
   [junit4]   2> 2666453 T7128 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 2666454 T7128 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2666456 T7183 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2666467 T7128 oasc.AbstractFullDistribZkTestBase.randomlyEnableAutoSoftCommit Not turning on auto soft commit
   [junit4]   2> 2666476 T7183 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2666480 T7183 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2666480 T7183 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C4939 name=collection1 org.apache.solr.core.SolrCore@7bb875df url=https://127.0.0.1:51031/collection1 node=127.0.0.1:51031_ C4939_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=https://127.0.0.1:51031, node_name=127.0.0.1:51031_, state=down}
   [junit4]   2> 2666480 T7183 C4939 P51031 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:51031/collection1/
   [junit4]   2> 2666480 T7183 C4939 P51031 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2666480 T7183 C4939 P51031 oasc.SyncStrategy.syncToMe https://127.0.0.1:51031/collection1/ has no replicas
   [junit4]   2> 2666481 T7183 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:51031/collection1/ shard1
   [junit4]   2> 2666481 T7183 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2666488 T7157 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2666499 T7157 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C4940 name=collection1 org.apache.solr.core.SolrCore@166ee827 url=https://127.0.0.1:51027/collection1 node=127.0.0.1:51027_ C4940_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=https://127.0.0.1:51027, node_name=127.0.0.1:51027_, state=active, leader=true}
   [junit4]   2> 2666509 T7148 C4940 P51027 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1479338620036841472)} 0 7
   [junit4]   2> 2666511 T7128 oascsi.CloudSolrServer.requestWithRetryOnStaleState ERROR Request to collection collection1 failed due to (510) org.apache.solr.common.SolrException: Not enough nodes to handle the request, retry? 0
   [junit4]   2> 2666518 T7128 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 51027
   [junit4]   2> 2666518 T7128 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=668295376
   [junit4]   2> 2666518 T7128 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 2666519 T7128 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2666520 T7157 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2666521 T7128 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@166ee827
   [junit4]   2> 2666522 T7153 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:51027",
   [junit4]   2> 	  "node_name":"127.0.0.1:51027_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 2666525 T7128 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=1,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 2666526 T7128 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 2666528 T7128 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2666528 T7128 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2666529 T7128 C4940 P51027 oasu.DirectUpdateHandler2.closeWriter Committing on IndexWriter close.
   [junit4]   2> 2666534 T7128 C4940 P51027 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/control/data/index,segFN=segments_2,generation=2}
   [junit4]   2> 2666535 T7128 C4940 P51027 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 2666537 T7128 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 2666537 T7128 oasc.CachingDirectoryFactory.close Closing StandardDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2666538 T7128 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/control/data/index [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/control/data/index;done=false>>]
   [junit4]   2> 2666538 T7128 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/control/data/index
   [junit4]   2> 2666539 T7128 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/control/data [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/control/data;done=false>>]
   [junit4]   2> 2666539 T7128 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/control/data
   [junit4]   2> 2666540 T7128 oasc.Overseer.close Overseer (id=92458663216218116-127.0.0.1:51027_-n_0000000000) closing
   [junit4]   2> 2666540 T7153 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:51027_
   [junit4]   2> 2666546 T7157 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2666552 T7185 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:51031_
   [junit4]   2> 2666552 T7185 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2666559 T7185 oasc.Overseer.start Overseer (id=92458663216218119-127.0.0.1:51031_-n_0000000001) starting
   [junit4]   2> 2666562 T7128 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 2666602 T7185 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 2666605 T7187 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2666622 T7186 oasc.Overseer$ClusterStateUpdater.run Replaying operations from work queue.
   [junit4]   2> 2666626 T7185 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2666648 T7186 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:51027",
   [junit4]   2> 	  "node_name":"127.0.0.1:51027_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 2666660 T7186 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2666703 T7183 oasc.ZkController.register We are https://127.0.0.1:51031/collection1/ and leader is https://127.0.0.1:51031/collection1/
   [junit4]   2> 2666703 T7183 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:51031
   [junit4]   2> 2666704 T7183 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2666705 T7183 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 2666705 T7183 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2666710 T7185 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2666730 T7186 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:51031",
   [junit4]   2> 	  "node_name":"127.0.0.1:51031_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 2666843 T7185 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2668045 T7184 oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read the data org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:347)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:344)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:344)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:346)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:89)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:296)
   [junit4]   2> 
   [junit4]   2> 2671552 T7128 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 51031
   [junit4]   2> 2671553 T7128 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1316883938
   [junit4]   2> 2671554 T7128 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 2671556 T7128 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2671561 T7128 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@7bb875df
   [junit4]   2> 2671561 T7185 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2671574 T7186 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:51031",
   [junit4]   2> 	  "node_name":"127.0.0.1:51031_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 2671578 T7128 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 2671578 T7128 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 2671578 T7128 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2671579 T7128 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2671579 T7185 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2671583 T7128 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 2671583 T7128 oasc.CachingDirectoryFactory.close Closing StandardDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2671584 T7128 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/jetty1/index [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/jetty1/index;done=false>>]
   [junit4]   2> 2671585 T7128 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/jetty1/index
   [junit4]   2> 2671585 T7128 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/jetty1 [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/jetty1;done=false>>]
   [junit4]   2> 2671585 T7128 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001/tempDir-001/jetty1
   [junit4]   2> 2671587 T7128 oasc.Overseer.close Overseer (id=92458663216218119-127.0.0.1:51031_-n_0000000001) closing
   [junit4]   2> 2671587 T7186 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:51031_
   [junit4]   2> 2671592 T7177 oaz.ClientCnxn$EventThread.processEvent ERROR Error while calling watcher  java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@1a8b885f rejected from java.util.concurrent.ThreadPoolExecutor@5f89056a[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 8]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2047)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
   [junit4]   2> 	at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:263)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 2673090 T7185 oascc.ZkStateReader.getIndividualColls WARN Error reading collections nodes org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:313)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:310)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:310)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getIndividualColls(ZkStateReader.java:483)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.constructState(ZkStateReader.java:448)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.access$200(ZkStateReader.java:55)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader$2.process(ZkStateReader.java:315)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3$1.run(SolrZkClient.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 2673091 T7185 oascc.ZkStateReader.getIndividualColls WARN Error reading collections nodes org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/control_collection/state.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:313)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:310)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:310)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getIndividualColls(ZkStateReader.java:483)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.constructState(ZkStateReader.java:448)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.access$200(ZkStateReader.java:55)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader$2.process(ZkStateReader.java:315)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3$1.run(SolrZkClient.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 2673093 T7189 oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read the data org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:347)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:344)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:344)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:346)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:89)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:296)
   [junit4]   2> 
   [junit4]   2> 2673115 T7128 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 2678101 T7128 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 2678107 T7128 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:51023 51023
   [junit4]   2> 2678286 T7129 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:51023 51023
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.method=testDistribSearch -Dtests.seed=9D91BDE760C23FDB -Dtests.slow=true -Dtests.locale=vi -Dtests.timezone=Pacific/Palau -Dtests.file.encoding=UTF-8
   [junit4] ERROR   20.3s | ChaosMonkeySafeLeaderTest.testDistribSearch <<<
   [junit4]    > Throwable #1: java.lang.NullPointerException
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9D91BDE760C23FDB:1C7733FF179D5FE7]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServer.requestWithRetryOnStaleState(CloudSolrServer.java:706)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:601)
   [junit4]    > 	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.deleteByQuery(SolrServer.java:285)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.deleteByQuery(SolrServer.java:271)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:728)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.tryDelete(ChaosMonkeySafeLeaderTest.java:194)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:112)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2678569 T7128 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest-9D91BDE760C23FDB-001
   [junit4]   2> NOTE: test params are: codec=Lucene410: {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=vi, timezone=Pacific/Palau
   [junit4]   2> NOTE: Mac OS X 10.8.5 x86_64/Oracle Corporation 1.8.0_20 (64-bit)/cpus=2,threads=1,free=319795856,total=461049856
   [junit4]   2> NOTE: All tests run in this JVM: [TestManagedSchema, DistributedQueryElevationComponentTest, MultiTermTest, SpellingQueryConverterTest, TestStandardQParsers, TestSimpleQParserPlugin, ReturnFieldsTest, PathHierarchyTokenizerFactoryTest, TestPostingsSolrHighlighter, TestRemoteStreaming, TestCodecSupport, BlockDirectoryTest, SolrPluginUtilsTest, TestDocSet, TestPartialUpdateDeduplication, UpdateParamsTest, MoreLikeThisHandlerTest, TimeZoneUtilsTest, TestValueSourceCache, MergeStrategyTest, ExternalCollectionsTest, TestReRankQParserPlugin, TestDefaultSearchFieldResource, SampleTest, RankQueryTest, TestQuerySenderListener, TestFieldResource, TestSolrCoreProperties, TestManagedSchemaDynamicFieldResource, NotRequiredUniqueKeyTest, TestIndexSearcher, BadComponentTest, PrimUtilsTest, FullSolrCloudDistribCmdsTest, TestManagedResource, TestShardHandlerFactory, TestClassNameShortening, FastVectorHighlighterTest, TestSolrQueryParserDefaultOperatorResource, TestMissingGroups, TestUniqueKeyFieldResource, OverriddenZkACLAndCredentialsProvidersTest, StressHdfsTest, HighlighterMaxOffsetTest, TestSortingResponseWriter, DirectUpdateHandlerOptimizeTest, TestCustomSort, HardAutoCommitTest, TestFieldTypeCollectionResource, SSLMigrationTest, TestRangeQuery, TestDistributedMissingSort, QueryParsingTest, ReplicationFactorTest, TestConfigSets, TestSolr4Spatial2, TestSolr4Spatial, TestDynamicFieldResource, RemoteQueryErrorTest, TestStressRecovery, TestManagedSchemaFieldTypeResource, TestJmxMonitoredMap, FieldAnalysisRequestHandlerTest, HdfsChaosMonkeySafeLeaderTest, CurrencyFieldXmlFileTest, TermsComponentTest, TestFaceting, InfoHandlerTest, TestBM25SimilarityFactory, HdfsSyncSliceTest, ScriptEngineTest, AddSchemaFieldsUpdateProcessorFactoryTest, DistributedQueryComponentOptimizationTest, QueryElevationComponentTest, CoreAdminRequestStatusTest, TestSolrIndexConfig, SolrRequestParserTest, TestCollationField, TestDefaultSimilarityFactory, DeleteLastCustomShardedReplicaTest, TestQueryTypes, TestBadConfig, TestReload, TestAddFieldRealTimeGet, OverseerTest, StatelessScriptUpdateProcessorFactoryTest, TestLeaderElectionZkExpiry, FieldFacetExtrasTest, TestCharFilters, DocValuesMissingTest, TestLFUCache, DeleteReplicaTest, CopyFieldTest, QueryResultKeyTest, TestComponentsName, SuggesterFSTTest, BasicZkTest, TestFieldCollectionResource, SuggesterTSTTest, TestNRTOpen, BasicFunctionalityTest, TestSolrQueryParserResource, HdfsUnloadDistributedZkTest, TestStressUserVersions, QueryEqualityTest, DocValuesTest, TestReloadAndDeleteDocs, HdfsWriteToMultipleCollectionsTest, SolrTestCaseJ4Test, CircularListTest, CoreAdminCreateDiscoverTest, PingRequestHandlerTest, BlockCacheTest, AnalysisErrorHandlingTest, LeaderElectionIntegrationTest, TestSurroundQueryParser, NumericFieldsTest, ExternalFileFieldSortTest, HighlighterTest, DistribDocExpirationUpdateProcessorTest, LegacyHTMLStripCharFilterTest, TestTolerantSearch, SoftAutoCommitTest, TestCloudPivotFacet, CursorPagingTest, TestSearcherReuse, TestGroupingSearch, TestHighFrequencyDictionaryFactory, PluginInfoTest, EnumFieldTest, RollingRestartTest, TestImplicitCoreProperties, TestShortCircuitedRequests, OverseerCollectionProcessorTest, AlternateDirectoryTest, PreAnalyzedFieldTest, TestCollapseQParserPlugin, LeaderElectionTest, SignatureUpdateProcessorFactoryTest, TestLRUCache, SolrCoreCheckLockOnStartupTest, RAMDirectoryFactoryTest, TestXIncludeConfig, TestHighlightDedupGrouping, SliceStateTest, SyncSliceTest, TestRealTimeGet, TestSort, TestFunctionQuery, TestMaxScoreQueryParser, TestStressVersions, DocExpirationUpdateProcessorFactoryTest, TestPerFieldSimilarity, CustomCollectionTest, TestFileDictionaryLookup, ShowFileRequestHandlerTest, TestAtomicUpdateErrorCases, IndexSchemaRuntimeFieldTest, BasicDistributedZk2Test, TestParamSet, TestDistributedGrouping, TestSolrDeletionPolicy1, TestManagedSynonymFilterFactory, CurrencyFieldOpenExchangeTest, RecoveryZkTest, TestMultiCoreConfBootstrap, SharedFSAutoReplicaFailoverTest, RequiredFieldsTest, TestExtendedDismaxParser, DistributedSuggestComponentTest, TestPhraseSuggestions, PolyFieldTest, TestCloudManagedSchema, DateMathParserTest, TestNonDefinedSimilarityFactory, HdfsCollectionsAPIDistributedZkTest, WordBreakSolrSpellCheckerTest, DistributedFacetPivotSmallTest, DocumentAnalysisRequestHandlerTest, TestHashPartitioner, TestLazyCores, SOLR749Test, HdfsLockFactoryTest, OverseerRolesTest, TestSchemaSimilarityResource, BasicDistributedZkTest, DOMUtilTest, SystemInfoHandlerTest, SpellPossibilityIteratorTest, EchoParamsTest, TestOrdValues, AbstractAnalyticsFacetTest, TermVectorComponentTest, TestRestManager, TestDynamicFieldCollectionResource, MBeansHandlerTest, TestRandomMergePolicy, BJQParserTest, TestNonNRTOpen, DistributedExpandComponentTest, HdfsBasicDistributedZkTest, TestSolrJ, TestNoOpRegenerator, TestPivotHelperCode, ShardRoutingCustomTest, DebugComponentTest, DistanceFunctionTest, TestUtils, HttpPartitionTest, OverseerStatusTest, TestCloudSchemaless, TestSolrDeletionPolicy2, BadCopyFieldTest, SolrCmdDistributorTest, ChaosMonkeySafeLeaderTest]
   [junit4] Completed in 20.31s, 1 test, 1 error <<< FAILURES!

[...truncated 686 lines...]
BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:491: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:471: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:60: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/extra-targets.xml:39: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:189: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:496: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1358: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:961: There were test failures: 438 suites, 1830 tests, 1 error, 50 ignored (22 assumptions)

Total time: 147 minutes 15 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.8.0) - Build # 1838 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/1838/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([177DFCBD6E4B9A3:809151D3A1BBD99F]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:550)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:583)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:205)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
	at sun.reflect.GeneratedMethodAccessor36.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11424 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/init-core-data-001
   [junit4]   2> 3304357 T8459 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 3304358 T8459 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /gzlpr/qe
   [junit4]   2> 3304365 T8459 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 3304367 T8459 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3304369 T8460 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 3304377 T8459 oasc.ZkTestServer.run start zk server on port:51398
   [junit4]   2> 3304379 T8459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 3304384 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3304391 T8466 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@373727ab name:ZooKeeperConnection Watcher:127.0.0.1:51398 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3304391 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3304392 T8459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 3304393 T8459 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 3304400 T8459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 3304402 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3304405 T8468 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f735d97 name:ZooKeeperConnection Watcher:127.0.0.1:51398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3304405 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3304406 T8459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 3304406 T8459 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 3304413 T8459 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 3304420 T8459 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 3304426 T8459 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 3304434 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 3304435 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 3304445 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 3304447 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 3304456 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 3304457 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 3304465 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 3304466 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 3304475 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 3304476 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 3304485 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 3304486 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 3304494 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 3304494 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 3304503 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 3304506 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 3304514 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 3304514 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 3304521 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 3304522 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 3304528 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 3304528 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 3304539 T8459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 3304541 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3304543 T8470 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5634ecfb name:ZooKeeperConnection Watcher:127.0.0.1:51398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3304544 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3304545 T8459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> Using old style solr.xml
   [junit4]   2> 3304549 T8459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 3304550 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3304554 T8472 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@43445b82 name:ZooKeeperConnection Watcher:127.0.0.1:51398 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3304554 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3304555 T8459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 3304555 T8459 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 3304563 T8459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 3304565 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3304568 T8474 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d40263e name:ZooKeeperConnection Watcher:127.0.0.1:51398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3304568 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3304569 T8459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 3304572 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf2/solrconfig.xml
   [junit4]   2> 3304574 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.xml
   [junit4]   2> 3304583 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
   [junit4]   2> 3304584 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf2/schema.xml
   [junit4]   2> 3304592 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 3304594 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 3304601 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
   [junit4]   2> 3304603 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf2/stopwords.txt
   [junit4]   2> 3304610 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
   [junit4]   2> 3304611 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf2/protwords.txt
   [junit4]   2> 3304620 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
   [junit4]   2> 3304621 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf2/currency.xml
   [junit4]   2> 3304629 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
   [junit4]   2> 3304630 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf2/open-exchange-rates.json
   [junit4]   2> 3304638 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 3304638 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 3304647 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
   [junit4]   2> 3304648 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf2/old_synonyms.txt
   [junit4]   2> 3304658 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
   [junit4]   2> 3304659 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf2/synonyms.txt
   [junit4]   2> 3304669 T8459 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
   [junit4]   2> 3304670 T8459 oascc.SolrZkClient.makePath makePath: /configs/conf2/elevate.xml
   [junit4]   2> 3306848 T8459 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 3306858 T8459 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 3306863 T8459 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:51404
   [junit4]   2> 3306867 T8459 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 3306867 T8459 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 3306868 T8459 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-002
   [junit4]   2> 3306868 T8459 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-002/'
   [junit4]   2> 3306918 T8459 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-002/solr.xml
   [junit4]   2> 3306930 T8459 oasc.CoreContainer.<init> New CoreContainer 1463402231
   [junit4]   2> 3306930 T8459 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-002/]
   [junit4]   2> 3306931 T8459 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 3306931 T8459 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 3306932 T8459 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 3306933 T8459 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 3306933 T8459 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 3306933 T8459 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 3306934 T8459 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 3306934 T8459 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 3306934 T8459 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 3306936 T8459 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
   [junit4]   2> 3306938 T8459 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 3306938 T8459 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 3306938 T8459 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 3306939 T8459 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51398/solr
   [junit4]   2> 3306939 T8459 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 3306939 T8459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 3306941 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3306944 T8485 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62c5cb5f name:ZooKeeperConnection Watcher:127.0.0.1:51398 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3306944 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3306945 T8459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 3306952 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3306955 T8487 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2440f293 name:ZooKeeperConnection Watcher:127.0.0.1:51398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3306956 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3306959 T8459 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 3306967 T8459 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 3306974 T8459 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 3306984 T8459 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 3306991 T8459 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 3307001 T8459 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 3307007 T8459 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51404_gzlpr%2Fqe
   [junit4]   2> 3307009 T8459 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51404_gzlpr%2Fqe
   [junit4]   2> 3307018 T8459 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 3307023 T8459 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 3307029 T8459 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 3307035 T8459 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:51404_gzlpr%2Fqe
   [junit4]   2> 3307035 T8459 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 3307043 T8459 oasc.Overseer.start Overseer (id=92480082912542726-127.0.0.1:51404_gzlpr%2Fqe-n_0000000000) starting
   [junit4]   2> 3307052 T8459 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 3307082 T8459 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 3307084 T8489 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 3307086 T8459 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 3307093 T8459 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 3307097 T8459 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3307108 T8488 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 3307113 T8459 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 3307113 T8459 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 3307117 T8459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 3307118 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3307121 T8493 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3dda9c6c name:ZooKeeperConnection Watcher:127.0.0.1:51398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3307121 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3307121 T8459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 3307124 T8459 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3307133 T8459 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 3307133 T8459 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 3307133 T8459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 3307135 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3307137 T8495 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5e2c0f98 name:ZooKeeperConnection Watcher:127.0.0.1:51398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3307137 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3307138 T8459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 3307143 T8496 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3307151 T8488 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: collection1
   [junit4]   2> 3307151 T8488 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 3307152 T8488 oasc.Overseer$ClusterStateUpdater.createCollection state version collection1 2
   [junit4]   2> 3307157 T8488 oasc.Overseer$ClusterStateUpdater.updateZkStates going to create_collection /collections/collection1/state.json
   [junit4]   2> 3307161 T8497 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3307162 T8496 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3309098 T8459 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 3309098 T8459 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 3309105 T8459 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 3309107 T8459 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:51409
   [junit4]   2> 3309110 T8459 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 3309110 T8459 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 3309111 T8459 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-003
   [junit4]   2> 3309111 T8459 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-003/'
   [junit4]   2> 3309166 T8459 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-003/solr.xml
   [junit4]   2> 3309188 T8459 oasc.CoreContainer.<init> New CoreContainer 137745080
   [junit4]   2> 3309189 T8459 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-003/]
   [junit4]   2> 3309190 T8459 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 3309190 T8459 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 3309191 T8459 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 3309191 T8459 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 3309192 T8459 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 3309193 T8459 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 3309193 T8459 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 3309194 T8459 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 3309194 T8459 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 3309197 T8459 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
   [junit4]   2> 3309199 T8459 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 3309200 T8459 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 3309201 T8459 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 3309201 T8459 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51398/solr
   [junit4]   2> 3309202 T8459 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 3309202 T8459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 3309204 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3309208 T8508 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a51babe name:ZooKeeperConnection Watcher:127.0.0.1:51398 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3309208 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3309209 T8459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 3309214 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3309217 T8510 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@543aa19b name:ZooKeeperConnection Watcher:127.0.0.1:51398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3309218 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3309238 T8459 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3310262 T8459 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51409_gzlpr%2Fqe
   [junit4]   2> 3310264 T8459 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51409_gzlpr%2Fqe
   [junit4]   2> 3310279 T8459 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 3310291 T8459 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 3310292 T8459 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 3312595 T8459 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 3312596 T8459 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 3312604 T8459 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 3312608 T8459 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:51412
   [junit4]   2> 3312613 T8459 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 3312614 T8459 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 3312615 T8459 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-004
   [junit4]   2> 3312615 T8459 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-004/'
   [junit4]   2> 3312679 T8459 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-004/solr.xml
   [junit4]   2> 3312732 T8459 oasc.CoreContainer.<init> New CoreContainer 856105501
   [junit4]   2> 3312732 T8459 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-004/]
   [junit4]   2> 3312733 T8459 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 3312733 T8459 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 3312733 T8459 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 3312733 T8459 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 3312734 T8459 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 3312734 T8459 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 3312734 T8459 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 3312734 T8459 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 3312736 T8459 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 3312744 T8459 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
   [junit4]   2> 3312750 T8459 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 3312751 T8459 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 3312752 T8459 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 3312753 T8459 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51398/solr
   [junit4]   2> 3312753 T8459 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 3312754 T8459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 3312756 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3312760 T8523 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7bbcd596 name:ZooKeeperConnection Watcher:127.0.0.1:51398 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3312760 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3312761 T8459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 3312774 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3312776 T8525 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a6887e0 name:ZooKeeperConnection Watcher:127.0.0.1:51398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3312777 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3312808 T8459 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3313830 T8459 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51412_gzlpr%2Fqe
   [junit4]   2> 3313832 T8459 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51412_gzlpr%2Fqe
   [junit4]   2> 3313847 T8459 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 3313862 T8459 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 3313862 T8459 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 3316090 T8459 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 3316092 T8459 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 3316102 T8459 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 3316108 T8459 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:51415
   [junit4]   2> 3316114 T8459 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 3316114 T8459 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 3316115 T8459 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-005
   [junit4]   2> 3316116 T8459 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-005/'
   [junit4]   2> 3316159 T8459 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-005/solr.xml
   [junit4]   2> 3316174 T8459 oasc.CoreContainer.<init> New CoreContainer 1466057745
   [junit4]   2> 3316175 T8459 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-005/]
   [junit4]   2> 3316175 T8459 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 3316176 T8459 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 3316176 T8459 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 3316177 T8459 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 3316177 T8459 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 3316177 T8459 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 3316178 T8459 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 3316178 T8459 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 3316179 T8459 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 3316181 T8459 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
   [junit4]   2> 3316182 T8459 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 3316183 T8459 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 3316183 T8459 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 3316183 T8459 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51398/solr
   [junit4]   2> 3316184 T8459 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 3316184 T8459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 3316185 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3316188 T8538 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@282288ec name:ZooKeeperConnection Watcher:127.0.0.1:51398 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3316188 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3316189 T8459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 3316200 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3316212 T8540 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@19bde358 name:ZooKeeperConnection Watcher:127.0.0.1:51398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3316213 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3316242 T8459 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3317267 T8459 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51415_gzlpr%2Fqe
   [junit4]   2> 3317270 T8459 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51415_gzlpr%2Fqe
   [junit4]   2> 3317286 T8459 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 3317298 T8459 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 3317298 T8459 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 3319345 T8459 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 4
   [junit4]   2> 3319346 T8459 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 3319355 T8459 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 3319357 T8459 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:51418
   [junit4]   2> 3319361 T8459 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 3319362 T8459 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 3319362 T8459 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-006
   [junit4]   2> 3319363 T8459 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-006/'
   [junit4]   2> 3319414 T8459 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-006/solr.xml
   [junit4]   2> 3319431 T8459 oasc.CoreContainer.<init> New CoreContainer 1698033424
   [junit4]   2> 3319432 T8459 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-006/]
   [junit4]   2> 3319432 T8459 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 3319433 T8459 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 3319434 T8459 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 3319434 T8459 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 3319434 T8459 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 3319434 T8459 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 3319435 T8459 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 3319435 T8459 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 3319436 T8459 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 3319438 T8459 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
   [junit4]   2> 3319441 T8459 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 3319441 T8459 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 3319442 T8459 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 3319442 T8459 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51398/solr
   [junit4]   2> 3319442 T8459 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 3319443 T8459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 3319447 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3319451 T8553 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62dbe5c1 name:ZooKeeperConnection Watcher:127.0.0.1:51398 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3319451 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3319452 T8459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 3319457 T8459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3319459 T8555 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6f2d72b4 name:ZooKeeperConnection Watcher:127.0.0.1:51398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3319459 T8459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3319480 T8459 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3320499 T8459 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51418_gzlpr%2Fqe
   [junit4]   2> 3320502 T8459 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51418_gzlpr%2Fqe
   [junit4]   2> 3320524 T8459 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 3320537 T8459 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 3320537 T8459 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 3320647 T8501 oasha.CollectionsHandler.handleCreateAction Creating Collection : replicationFactor=2&collection.configName=conf1&router.field=myOwnField&name=solrj_collection&action=CREATE&numShards=2&wt=javabin&version=2
   [junit4]   2> 3320652 T8496 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3320658 T8489 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"solrj_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "collection.configName":"conf1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "router.field":"myOwnField"}
   [junit4]   2> 3320660 T8558 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : create , {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"solrj_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "collection.configName":"conf1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "router.field":"myOwnField"}
   [junit4]   2> 3320666 T8558 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection 
   [junit4]   2> 3320670 T8558 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
   [junit4]   2> 3320716 T8496 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3320749 T8488 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
   [junit4]   2> 3320749 T8488 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
   [junit4]   2> 3320750 T8488 oasc.Overseer$ClusterStateUpdater.createCollection state version solrj_collection 1
   [junit4]   2> 3320758 T8497 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 3320758 T8511 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 3320759 T8526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 3320760 T8541 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 3320761 T8556 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 3320762 T8496 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 3320818 T8558 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection solrj_collection, shardNames [shard1, shard2] , replicationFactor : 2
   [junit4]   2> 3320819 T8558 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:51404_gzlpr%2Fqe
   [junit4]   2> 3320823 T8558 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:51418_gzlpr%2Fqe
   [junit4]   2> 3320833 T8558 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:51409_gzlpr%2Fqe
   [junit4]   2> 3320882 T8483 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection.configName=conf1&name=solrj_collection_shard1_replica1&action=CREATE&numShards=2&collection=solrj_collection&shard=shard1&wt=javabin&version=2
   [junit4]   2> 3320886 T8558 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:51415_gzlpr%2Fqe
   [junit4]   2> 3320925 T8551 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection.configName=conf1&name=solrj_collection_shard1_replica2&action=CREATE&numShards=2&collection=solrj_collection&shard=shard1&wt=javabin&version=2
   [junit4]   2> 3321000 T8551 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down collection=solrj_collection
   [junit4]   2> 3321004 T8551 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 3321006 T8532 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection.configName=conf1&name=solrj_collection_shard2_replica2&action=CREATE&numShards=2&collection=solrj_collection&shard=shard2&wt=javabin&version=2
   [junit4]   2> 3321017 T8496 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3321022 T8483 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down collection=solrj_collection
   [junit4]   2> 3321026 T8505 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection.configName=conf1&name=solrj_collection_shard2_replica1&action=CREATE&numShards=2&collection=solrj_collection&shard=shard2&wt=javabin&version=2
   [junit4]   2> 3321031 T8532 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down collection=solrj_collection
   [junit4]   2> 3321035 T8505 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down collection=solrj_collection
   [junit4]   2> 3321036 T8532 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 3321037 T8505 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 3321039 T8483 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 3321085 T8488 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"solrj_collection_shard1_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:51418/gzlpr/qe",
   [junit4]   2> 	  "node_name":"127.0.0.1:51418_gzlpr%2Fqe",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 3321105 T8488 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"solrj_collection_shard1_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:51404/gzlpr/qe",
   [junit4]   2> 	  "node_name":"127.0.0.1:51404_gzlpr%2Fqe",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 3321114 T8496 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3321127 T8488 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"solrj_collection_shard2_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:51415/gzlpr/qe",
   [junit4]   2> 	  "node_name":"127.0.0.1:51415_gzlpr%2Fqe",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 3321133 T8496 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3321138 T8488 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"solrj_collection_shard2_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:51409/gzlpr/qe",
   [junit4]   2> 	  "node_name":"127.0.0.1:51409_gzlpr%2Fqe",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 3321146 T8496 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3321274 T8511 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 3321274 T8497 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 3321274 T8541 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 3321276 T8556 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 3321276 T8526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 3321279 T8496 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 3322012 T8551 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
   [junit4]   2> 3322014 T8551 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 3322015 T8551 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 3322016 T8551 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 3322018 T8551 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 3322018 T8551 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-006/solrj_collection_shard1_replica2/'
   [junit4]   2> 3322040 T8532 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
   [junit4]   2> 3322041 T8532 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 3322041 T8483 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
   [junit4]   2> 3322042 T8483 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 3322044 T8505 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
   [junit4]   2> 3322044 T8505 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 3322048 T8532 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 3322048 T8532 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 3322052 T8505 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 3322052 T8505 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 3322055 T8505 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 3322056 T8505 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-003/solrj_collection_shard2_replica1/'
   [junit4]   2> 3322065 T8532 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 3322065 T8532 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-005/solrj_collection_shard2_replica2/'
   [junit4]   2> 3322076 T8483 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 3322076 T8483 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 3322109 T8483 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 3322112 T8483 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-002/solrj_collection_shard1_replica1/'
   [junit4]   2> 3322162 T8551 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 3322194 T8551 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 3322199 T8551 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 3322206 T8505 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 3322207 T8551 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
   [junit4]   2> 3322231 T8532 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 3322241 T8505 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 3322281 T8532 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 3322286 T8505 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 3322296 T8532 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 3322307 T8483 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 3322321 T8483 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 3322323 T8532 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
   [junit4]   2> 3322326 T8505 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
   [junit4]   2> 3322335 T8483 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 3322344 T8483 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
   [junit4]   2> 3322645 T8551 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 3322650 T8505 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 3322679 T8505 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 3322681 T8505 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 3322686 T8551 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 3322688 T8551 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 3322724 T8532 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 3322733 T8532 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 3322735 T8532 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 3322747 T8505 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 3322752 T8551 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 3322753 T8505 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 3322761 T8505 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 3322761 T8551 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 3322765 T8505 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 3322766 T8505 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 3322767 T8505 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 3322769 T8551 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 3322772 T8505 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 3322779 T8505 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 3322779 T8505 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica1' using configuration from collection solrj_collection
   [junit4]   2> 3322780 T8505 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 3322781 T8505 oasc.SolrCore.<init> [solrj_collection_shard2_replica1] Opening new SolrCore at /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-003/solrj_collection_shard2_replica1/, dataDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-003/solrj_collection_shard2_replica1/data/
   [junit4]   2> 3322781 T8505 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7663b62b
   [junit4]   2> 3322782 T8505 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-003/solrj_collection_shard2_replica1/data
   [junit4]   2> 3322782 T8505 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-003/solrj_collection_shard2_replica1/data/index/
   [junit4]   2> 3322783 T8505 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica1] Solr index directory '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-003/solrj_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 3322774 T8551 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 3322785 T8551 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 3322785 T8551 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 3322784 T8505 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-003/solrj_collection_shard2_replica1/data/index
   [junit4]   2> 3322788 T8505 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 3322789 T8505 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-c8b1528)),segFN=segments_1,generation=1}
   [junit4]   2> 3322790 T8505 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 3322793 T8505 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 3322795 T8505 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 3322795 T8505 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 3322801 T8505 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 3322796 T8551 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 3322804 T8551 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 3322804 T8551 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using configuration from collection solrj_collection
   [junit4]   2> 3322805 T8551 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 3322806 T8551 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-006/solrj_collection_shard1_replica2/, dataDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-006/solrj_collection_shard1_replica2/data/
   [junit4]   2> 3322806 T8551 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7663b62b
   [junit4]   2> 3322803 T8505 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 3322809 T8551 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-006/solrj_collection_shard1_replica2/data
   [junit4]   2> 3322810 T8551 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-006/solrj_collection_shard1_replica2/data/index/
   [junit4]   2> 3322810 T8551 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-006/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 3322811 T8551 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-006/solrj_collection_shard1_replica2/data/index
   [junit4]   2> 3322812 T8551 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 3322813 T8551 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-78bd2325)),segFN=segments_1,generation=1}
   [junit4]   2> 3322815 T8551 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 3322810 T8505 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 3322817 T8505 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 3322818 T8505 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 3322818 T8505 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 3322819 T8505 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 3322819 T8505 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 3322820 T8505 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 3322820 T8551 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 3322822 T8551 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 3322805 T8483 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 3322827 T8532 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 3322823 T8551 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 3322829 T8551 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 3322829 T8551 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 3322829 T8551 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 3322830 T8551 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 3322831 T8551 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 3322831 T8551 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 3322832 T8551 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 3322833 T8551 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 3322833 T8551 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 3322834 T8551 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 3322835 T8551 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 3322835 T8551 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 3322835 T8551 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 3322836 T8551 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 3322839 T8551 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 3322820 T8505 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 3322844 T8505 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 3322844 T8505 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 3322845 T8505 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 3322846 T8505 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 3322847 T8505 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 3322849 T8505 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 3322850 T8505 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 3322827 T8483 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 3322855 T8483 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 3322845 T8551 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 3322844 T8532 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 3322865 T8551 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 3322869 T8551 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 3322872 T8551 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 3322875 T8505 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 3322878 T8505 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 3322880 T8532 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 3322881 T8505 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 3322883 T8532 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 3322884 T8532 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 3322885 T8532 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 3322886 T8551 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 3322889 T8532 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 3322889 T8532 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 3322889 T8551 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 3322890 T8532 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica2' using configuration from collection solrj_collection
   [junit4]   2> 3322894 T8532 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 3322894 T8532 oasc.SolrCore.<init> [solrj_collection_shard2_replica2] Opening new SolrCore at ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-005/solrj_collection_shard2_replica2/, dataDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-005/solrj_collection_shard2_replica2/data/
   [junit4]   2> 3322895 T8532 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7663b62b
   [junit4]   2> 3322896 T8551 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 3322896 T8551 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 3322896 T8532 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-005/solrj_collection_shard2_replica2/data
   [junit4]   2> 3322899 T8532 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-005/solrj_collection_shard2_replica2/data/index/
   [junit4]   2> 3322900 T8532 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica2] Solr index directory '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-005/solrj_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 3322901 T8532 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001/tempDir-005/solrj_collection_shard2_replica2/data/index
   [junit4]   2> 3322901 T8532 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 3322902 T8532 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-2bd81d60)),segFN=segments_1,generation=1}
   [junit4]   2> 3322903 T8532 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 3322897 T8551 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=83.6044921875, floorSegmentMB=0.560546875, forceMergeDeletesPctAllowed=17.955457698485574, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4779036779837921
   [junit4]   2> 3322908 T8551 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-78bd2325)),segFN=segments_1,generation=1}
   [junit4]   2> 3322908 T8551 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 3322909 T8551 oass.SolrIndexSearcher.<init> Opening Searcher@768773da[solrj_collection_shard1_replica2] main
   [junit4]   2> 3322909 T8551 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 3322898 T8505 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 3322916 T8551 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 3322918 T8551 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 3322920 T8551 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 3322921 T8551 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 3322922 T8551 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 3322926 T8505 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 3322927 T8505 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 3322928 T8505 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=83.6044921875, floorSegmentMB=0.560546875, forceMergeDeletesPctAllowed=17.955457698485574, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4779036779837921
   [junit4]   2> 3322929 T8551 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 3322929 T8551 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 3322931 T8551 oascc.SolrZkClient.makePath makePath: /configs/conf1/_rest_managed.json
   [junit4]   2> 3322938 T8532 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 3322939 T8505 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-c8b1528)),segFN=segments_1,generation=1}
   [junit4]   2> 3322940 T8505 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 3322941 T8505 oass.SolrIndexSearcher.<init> Opening Searcher@408f507b[solrj_collection_shard2_replica1] main
   [junit4]   2> 3322941 T8505 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 3322939 T8551 oasr.ManagedResourceStorage$ZooKeeperStorageIO$1.close Wrote 38 bytes to new znode /configs/conf1/_rest_managed.json
   [junit4]   2> 3322943 T8551 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 3322943 T8551 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 3322946 T8505 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 3322946 T8505 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 3322949 T8505 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 3322949 T8505 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 3322950 T8505 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 3322951 T8483 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 3322954 T8505 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream Read 38 bytes from znode /configs/conf1/_rest_managed.json
   [junit4]   2> 3322954 T8505 oasr.ManagedResourceStorage.load Loaded LinkedHashMap at path _rest_managed.json using ZooKeeperStorageIO:path=/

[...truncated too long message...]

.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:344)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:383)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:126)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:333)
   [junit4]   2> 
   [junit4]   2> 4080309 T8459 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/gzlpr/qe,null}
   [junit4]   2> 4085300 T8459 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 4085304 T8459 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:51398 51398
   [junit4]   2> 4085324 T8460 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:51398 51398
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CollectionsAPIDistributedZkTest -Dtests.method=testDistribSearch -Dtests.seed=177DFCBD6E4B9A3 -Dtests.slow=true -Dtests.locale=fr_FR -Dtests.timezone=Australia/North -Dtests.file.encoding=UTF-8
   [junit4] ERROR    781s | CollectionsAPIDistributedZkTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([177DFCBD6E4B9A3:809151D3A1BBD99F]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:550)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:583)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:205)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 4085683 T8459 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-177DFCBD6E4B9A3-001
   [junit4]   2> 781354 T8458 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene410: {other_tl1=PostingsFormat(name=Memory doPackFST= false), _version_=PostingsFormat(name=Memory doPackFST= false), multiDefault=PostingsFormat(name=SimpleText), a_t=PostingsFormat(name=Memory doPackFST= false), intDefault=PostingsFormat(name=Memory doPackFST= false), a_i1=PostingsFormat(name=Memory doPackFST= false), id=PostingsFormat(name=MockRandom), text=PostingsFormat(name=Lucene41VarGapFixedInterval), range_facet_l=PostingsFormat(name=Memory doPackFST= false), timestamp=PostingsFormat(name=Memory doPackFST= false)}, docValues:{timestamp=DocValuesFormat(name=Lucene410)}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=fr_FR, timezone=Australia/North
   [junit4]   2> NOTE: Mac OS X 10.8.5 x86_64/Oracle Corporation 1.8.0_20 (64-bit)/cpus=2,threads=1,free=87921712,total=342777856
   [junit4]   2> NOTE: All tests run in this JVM: [TestReRankQParserPlugin, TestManagedSchemaFieldResource, TestUniqueKeyFieldResource, OverseerTest, DocValuesMultiTest, TestGroupingSearch, ChaosMonkeySafeLeaderTest, TestUpdate, TestChildDocTransformer, TestCollationFieldDocValues, TestFiltering, DirectUpdateHandlerTest, TestSearcherReuse, ReturnFieldsTest, MinimalSchemaTest, PluginInfoTest, DeleteLastCustomShardedReplicaTest, TestMultiCoreConfBootstrap, DocumentAnalysisRequestHandlerTest, TestSolrJ, CoreAdminRequestStatusTest, TestPerFieldSimilarity, QueryParsingTest, AlternateDirectoryTest, BasicFunctionalityTest, DocValuesTest, HighlighterMaxOffsetTest, UpdateParamsTest, TestDocSet, HdfsSyncSliceTest, InfoHandlerTest, FullSolrCloudDistribCmdsTest, SuggesterTSTTest, SSLMigrationTest, FileUtilsTest, ZkSolrClientTest, SpatialFilterTest, TermVectorComponentDistributedTest, HdfsBasicDistributedZk2Test, TestCopyFieldCollectionResource, CurrencyFieldOpenExchangeTest, UUIDFieldTest, NumericFieldsTest, StandardRequestHandlerTest, TestIntervalFaceting, TestCursorMarkWithoutUniqueKey, TestWriterPerf, DeleteShardTest, TestBM25SimilarityFactory, TestDistributedMissingSort, HdfsLockFactoryTest, CustomCollectionTest, TestElisionMultitermQuery, TestLeaderElectionZkExpiry, TriLevelCompositeIdRoutingTest, HighlighterConfigTest, OpenCloseCoreStressTest, TestPhraseSuggestions, TestLazyCores, TestSolr4Spatial2, TestLMDirichletSimilarityFactory, SolrIndexSplitterTest, TestSolrQueryParserResource, DeleteReplicaTest, FieldAnalysisRequestHandlerTest, SolrCoreCheckLockOnStartupTest, TestPartialUpdateDeduplication, TestFieldTypeResource, DocumentBuilderTest, TestIndexSearcher, TestPostingsSolrHighlighter, TestPHPSerializedResponseWriter, HttpPartitionTest, ShardRoutingCustomTest, TestFuzzyAnalyzedSuggestions, TestDynamicFieldResource, TestRealTimeGet, RequiredFieldsTest, BasicDistributedZkTest, TestRandomDVFaceting, PrimUtilsTest, TestBadConfig, TestCloudSchemaless, TestExceedMaxTermLength, SoftAutoCommitTest, TestCollapseQParserPlugin, SpellPossibilityIteratorTest, TestSuggestSpellingConverter, TestQueryUtils, TestCloudManagedSchemaConcurrent, RecoveryZkTest, SortByFunctionTest, OverseerRolesTest, TestPivotHelperCode, ShardRoutingTest, TestAtomicUpdateErrorCases, TestSchemaNameResource, CoreContainerCoreInitFailuresTest, TestInitQParser, XmlUpdateRequestHandlerTest, ChangedSchemaMergeTest, TestCloudManagedSchema, TestMaxScoreQueryParser, XsltUpdateRequestHandlerTest, TestSolrDeletionPolicy2, TestWordDelimiterFilterFactory, TestOmitPositions, TestSolr4Spatial, ChaosMonkeyNothingIsSafeTest, JSONWriterTest, TestManagedResource, TestOrdValues, ModifyConfFileTest, TestBinaryResponseWriter, TestJmxIntegration, TestFoldingMultitermQuery, TestFieldTypeCollectionResource, TestLuceneMatchVersion, RollingRestartTest, RAMDirectoryFactoryTest, DistributedSuggestComponentTest, ZkCLITest, TestRawResponseWriter, TestBinaryField, SolrIndexConfigTest, TestStandardQParsers, TestComponentsName, TestCSVResponseWriter, TestStressVersions, TestUtils, TestQueryTypes, SolrCmdDistributorTest, TestRemoteStreaming, TestAnalyzedSuggestions, TestSort, SliceStateTest, AddSchemaFieldsUpdateProcessorFactoryTest, QueryResultKeyTest, TestSchemaResource, DistributedDebugComponentTest, HardAutoCommitTest, TestCustomSort, HdfsDirectoryTest, SolrInfoMBeanTest, TestFastOutputStream, TestStressLucene, TestSolrXmlPersistence, TestCloudPivotFacet, TestSearchPerf, TestRestManager, CircularListTest, IndexSchemaRuntimeFieldTest, DistributedQueryElevationComponentTest, DistributedTermsComponentTest, OverseerStatusTest, DirectSolrSpellCheckerTest, RequestHandlersTest, IndexBasedSpellCheckerTest, HighlighterTest, BadIndexSchemaTest, AssignTest, TestShortCircuitedRequests, SchemaVersionSpecificBehaviorTest, TestSortingResponseWriter, DateMathParserTest, TestCodecSupport, ZkControllerTest, PeerSyncTest, SolrTestCaseJ4Test, TestNonNRTOpen, TestRandomFaceting, TestImplicitCoreProperties, TestCloudInspectUtil, DOMUtilTest, TestSweetSpotSimilarityFactory, NoCacheHeaderTest, SyncSliceTest, PreAnalyzedFieldTest, TestFieldSortValues, RemoteQueryErrorTest, DistributedQueryComponentCustomSortTest, HdfsUnloadDistributedZkTest, TestArbitraryIndexDir, TestSolrCoreProperties, TestCollectionAPI, ExternalFileFieldSortTest, QueryElevationComponentTest, TestManagedSchema, SignatureUpdateProcessorFactoryTest, UpdateRequestProcessorFactoryTest, TestMissingGroups, PingRequestHandlerTest, TestQuerySenderNoQuery, HdfsRecoveryZkTest, TestFastLRUCache, HdfsWriteToMultipleCollectionsTest, TestRandomMergePolicy, TestManagedSynonymFilterFactory, SystemInfoHandlerTest, TestValueSourceCache, TestStressReorder, TestCSVLoader, ClusterStateTest, DateRangeFieldTest, CoreAdminHandlerTest, CachingDirectoryFactoryTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest]
   [junit4] Completed in 781.65s, 1 test, 1 error <<< FAILURES!

[...truncated 660 lines...]
BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:491: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:471: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:60: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/extra-targets.xml:39: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:189: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:496: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1358: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:961: There were test failures: 429 suites, 1772 tests, 1 error, 50 ignored (22 assumptions)

Total time: 163 minutes 20 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.8.0) - Build # 1837 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/1837/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

All tests passed

Build Log:
[...truncated 70576 lines...]
BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:491: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:78: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:548: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:554: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1959: Tidy was unable to process file /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/docs/solr-core/org/apache/solr/core/InitParams.html, 1 returned.

Total time: 223 minutes 14 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.7.0) - Build # 1836 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/1836/
Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseG1GC

All tests passed

Build Log:
[...truncated 60132 lines...]
-documentation-lint:
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec] 
     [exec] Crawl/parse...
     [exec] 
     [exec] Verify...
     [echo] Checking for malformed docs...
     [exec] 
     [exec] /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/docs/solr-analytics/overview-summary.html
     [exec]   missing: org.apache.solr.handler.component
     [exec] 
     [exec] Missing javadocs were found!

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:491: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:78: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:548: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:564: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:2471: exec returned: 1

Total time: 228 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.7.0) - Build # 1835 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/1835/
Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseG1GC

All tests passed

Build Log:
[...truncated 55618 lines...]
  [javadoc] Generating Javadoc
  [javadoc] Javadoc execution
  [javadoc] Loading source files for package org.apache.solr.analytics.accumulator...
  [javadoc] Loading source files for package org.apache.solr.analytics.accumulator.facet...
  [javadoc] Loading source files for package org.apache.solr.analytics.expression...
  [javadoc] Loading source files for package org.apache.solr.analytics.plugin...
  [javadoc] Loading source files for package org.apache.solr.analytics.request...
  [javadoc] Loading source files for package org.apache.solr.analytics.statistics...
  [javadoc] Loading source files for package org.apache.solr.analytics.util...
  [javadoc] Loading source files for package org.apache.solr.analytics.util.valuesource...
  [javadoc] Loading source files for package org.apache.solr.handler.component...
  [javadoc] Constructing Javadoc information...
  [javadoc] Standard Doclet version 1.7.0_67
  [javadoc] Building tree for all the packages and classes...
  [javadoc] Building index for all the packages and classes...
  [javadoc] javadoc: error - Error while reading file /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/contrib/analytics/src/java/overview.html
  [javadoc] Building index for all classes...
  [javadoc] Generating /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/docs/solr-analytics/help-doc.html...
  [javadoc] Note: Custom tags that were not seen:  @lucene.internal, @lucene.experimental
  [javadoc] 1 error

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:491: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:78: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:548: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:463: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:440: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:243: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:291: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:2001: Javadoc returned 1

Total time: 226 minutes 2 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.8.0) - Build # 1834 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/1834/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.client.solrj.TestLBHttpSolrServer.testReliability

Error Message:
No live SolrServers available to handle this request

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
	at __randomizedtesting.SeedInfo.seed([2E5BBD83E82DC359:EF9360C5494B12F0]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:528)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.client.solrj.TestLBHttpSolrServer.testReliability(TestLBHttpSolrServer.java:223)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:53542/solr
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:560)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:481)
	... 43 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:150)
	at java.net.SocketInputStream.read(SocketInputStream.java:121)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:448)
	... 46 more




Build Log:
[...truncated 12317 lines...]
   [junit4] Suite: org.apache.solr.client.solrj.TestLBHttpSolrServer
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/init-core-data-001
   [junit4]   2> 15029 T31 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 15036 T31 oas.SolrTestCaseJ4.setUp ###Starting testReliability
   [junit4]   2> 15719 T31 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 16110 T31 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53538
   [junit4]   2> 16195 T31 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 16209 T31 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 16209 T31 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001
   [junit4]   2> 16209 T31 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/'
   [junit4]   2> 16252 T31 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/solr.xml
   [junit4]   2> 16309 T31 oasc.CoreContainer.<init> New CoreContainer 147469593
   [junit4]   2> 16310 T31 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/]
   [junit4]   2> 16427 T31 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 16427 T31 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 16428 T31 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 16428 T31 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 16430 T31 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 16430 T31 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 16430 T31 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 16430 T31 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 16431 T31 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 16438 T31 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 16439 T31 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 16439 T31 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 16439 T31 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 16462 T43 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/'
   [junit4]   2> 16543 T43 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 16578 T43 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 16584 T43 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 16589 T43 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 16620 T43 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 16621 T43 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/
   [junit4]   2> 16621 T43 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockFSDirectoryFactory
   [junit4]   2> 16624 T43 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/, dataDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/
   [junit4]   2> 16624 T43 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 16682 T43 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data
   [junit4]   2> 16684 T43 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index/
   [junit4]   2> 16685 T43 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 16706 T43 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index
   [junit4]   2> 16734 T43 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 16735 T43 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 16738 T43 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 16739 T43 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 16740 T43 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 16740 T43 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 16741 T43 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 16741 T43 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 16742 T43 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 16746 T43 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 16747 T43 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 16840 T43 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 16847 T43 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 16849 T43 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 16851 T43 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 16852 T43 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 16854 T43 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 16854 T43 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 16911 T43 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 16911 T43 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 16912 T43 oass.SolrIndexSearcher.<init> Opening Searcher@4be867ad[collection1] main
   [junit4]   2> 16913 T43 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/conf
   [junit4]   2> 16913 T43 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/conf}
   [junit4]   2> 16913 T43 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/conf
   [junit4]   2> 16914 T43 oasr.ManagedResource.reloadFromStorage WARN No stored data found for /rest/managed
   [junit4]   2> 16916 T43 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using file:dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/conf
   [junit4]   2> 16916 T43 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 16937 T43 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 16937 T43 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 16937 T44 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4be867ad[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 16941 T31 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0
   [junit4]   2> 16941 T31 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> ASYNC  NEW_CORE C0 name=collection1 org.apache.solr.core.SolrCore@701f8f5b
   [junit4]   2> 19477 T41 C0 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]} 0 1119
   [junit4]   2> ASYNC  NEW_CORE C1 name=collection1 org.apache.solr.core.SolrCore@701f8f5b
   [junit4]   2> 19654 T40 C1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 20288 T40 C1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index),segFN=segments_2,generation=2}
   [junit4]   2> 20290 T40 C1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> ASYNC  NEW_CORE C2 name=collection1 org.apache.solr.core.SolrCore@701f8f5b
   [junit4]   2> 20541 T40 C2 oass.SolrIndexSearcher.<init> Opening Searcher@7a345714[collection1] main
   [junit4]   2> 20547 T40 C2 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 20557 T44 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a345714[collection1] main{UninvertingDirectoryReader(Uninverting(_0(5.0.0):c10))}
   [junit4]   2> 20558 T40 C2 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} {commit=} 0 905
   [junit4]   2> 20578 T31 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 20586 T31 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53540
   [junit4]   2> 20587 T31 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 20588 T31 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 20588 T31 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001
   [junit4]   2> 20589 T31 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/'
   [junit4]   2> 20638 T31 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/solr.xml
   [junit4]   2> 20677 T31 oasc.CoreContainer.<init> New CoreContainer 510145216
   [junit4]   2> 20678 T31 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/]
   [junit4]   2> 20683 T31 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 20683 T31 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 20684 T31 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 20684 T31 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 20684 T31 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 20685 T31 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 20686 T31 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 20686 T31 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 20687 T31 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 20687 T31 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 20688 T31 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 20688 T31 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 20689 T31 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 20697 T55 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/'
   [junit4]   2> 20735 T55 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 20752 T55 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 20754 T55 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 20762 T55 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 20797 T55 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 20797 T55 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/
   [junit4]   2> 20798 T55 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockFSDirectoryFactory
   [junit4]   2> 20799 T55 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/, dataDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/
   [junit4]   2> 20800 T55 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 20805 T55 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data
   [junit4]   2> 20807 T55 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index/
   [junit4]   2> 20807 T55 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 20811 T55 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index
   [junit4]   2> 20826 T55 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 20826 T55 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 20854 T55 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 20855 T55 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 20855 T55 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 20855 T55 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 20856 T55 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 20856 T55 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 20856 T55 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 20857 T55 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 20858 T55 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 20864 T55 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 20888 T55 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 20890 T55 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 20901 T55 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 20902 T55 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 20904 T55 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 20905 T55 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 20912 T55 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 20913 T55 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 20914 T55 oass.SolrIndexSearcher.<init> Opening Searcher@689ec4fe[collection1] main
   [junit4]   2> 20914 T55 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/conf
   [junit4]   2> 20914 T55 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/conf}
   [junit4]   2> 20915 T55 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/conf
   [junit4]   2> 20916 T55 oasr.ManagedResource.reloadFromStorage WARN No stored data found for /rest/managed
   [junit4]   2> 20917 T55 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using file:dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/conf
   [junit4]   2> 20918 T55 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 20918 T55 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 20919 T56 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@689ec4fe[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 20920 T55 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 20923 T31 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0
   [junit4]   2> 20924 T31 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> ASYNC  NEW_CORE C3 name=collection1 org.apache.solr.core.SolrCore@39299d2f
   [junit4]   2> 20938 T49 C3 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]} 0 7
   [junit4]   2> 20944 T50 C3 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> ASYNC  NEW_CORE C4 name=collection1 org.apache.solr.core.SolrCore@39299d2f
   [junit4]   2> 20996 T50 C4 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index),segFN=segments_2,generation=2}
   [junit4]   2> 21000 T50 C4 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 21043 T50 C4 oass.SolrIndexSearcher.<init> Opening Searcher@462b09df[collection1] main
   [junit4]   2> 21043 T50 C4 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 21045 T56 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@462b09df[collection1] main{UninvertingDirectoryReader(Uninverting(_0(5.0.0):c10))}
   [junit4]   2> 21048 T50 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} {commit=} 0 104
   [junit4]   2> 21069 T31 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 21076 T31 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53542
   [junit4]   2> 21078 T31 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 21078 T31 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 21078 T31 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001
   [junit4]   2> 21078 T31 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/'
   [junit4]   2> 21134 T31 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/solr.xml
   [junit4]   2> 21166 T31 oasc.CoreContainer.<init> New CoreContainer 153560465
   [junit4]   2> 21166 T31 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/]
   [junit4]   2> 21168 T31 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 21168 T31 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 21169 T31 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 21169 T31 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 21169 T31 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 21169 T31 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 21170 T31 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 21170 T31 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 21171 T31 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 21171 T31 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 21173 T31 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 21174 T31 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 21174 T31 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 21182 T67 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/'
   [junit4]   2> 21217 T67 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 21243 T67 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 21244 T67 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 21260 T67 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 21268 T67 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 21269 T67 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/
   [junit4]   2> 21269 T67 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockFSDirectoryFactory
   [junit4]   2> 21270 T67 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/, dataDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/
   [junit4]   2> 21270 T67 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 21289 T67 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data
   [junit4]   2> 21298 T67 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index/
   [junit4]   2> 21300 T67 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 21302 T67 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index
   [junit4]   2> 21308 T67 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 21309 T67 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 21313 T67 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 21313 T67 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 21313 T67 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 21314 T67 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 21314 T67 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 21315 T67 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 21315 T67 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 21315 T67 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 21316 T67 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 21316 T67 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 21354 T67 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 21356 T67 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 21357 T67 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 21359 T67 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 21360 T67 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 21361 T67 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 21393 T67 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 21393 T67 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 21394 T67 oass.SolrIndexSearcher.<init> Opening Searcher@3f33b98c[collection1] main
   [junit4]   2> 21395 T67 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/conf
   [junit4]   2> 21395 T67 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/conf}
   [junit4]   2> 21395 T67 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/conf
   [junit4]   2> 21396 T67 oasr.ManagedResource.reloadFromStorage WARN No stored data found for /rest/managed
   [junit4]   2> 21397 T67 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using file:dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/conf
   [junit4]   2> 21397 T67 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 21398 T67 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 21398 T67 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 21398 T68 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3f33b98c[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 21400 T31 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0
   [junit4]   2> 21400 T31 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> ASYNC  NEW_CORE C5 name=collection1 org.apache.solr.core.SolrCore@76ad079c
   [junit4]   2> 21413 T61 C5 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]} 0 9
   [junit4]   2> 21428 T62 C5 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 21469 T62 C5 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index),segFN=segments_2,generation=2}
   [junit4]   2> 21471 T62 C5 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 21482 T62 C5 oass.SolrIndexSearcher.<init> Opening Searcher@fdea7f8[collection1] main
   [junit4]   2> 21483 T68 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@fdea7f8[collection1] main{UninvertingDirectoryReader(Uninverting(_0(5.0.0):c10))}
   [junit4]   2> 21484 T62 C5 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 21484 T62 C5 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} {commit=} 0 58
   [junit4]   2> 21503 T31 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=510145216
   [junit4]   2> 21509 T31 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@39299d2f
   [junit4]   2> 21511 T31 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=10,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 21516 T31 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 21516 T31 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 21516 T31 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 21526 T31 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 21536 T31 oasc.CachingDirectoryFactory.close Closing MockFSDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 21537 T31 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index;done=false>>]
   [junit4]   2> 21537 T31 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index
   [junit4]   2> 21968 T31 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data;done=false>>]
   [junit4]   2> 21969 T31 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data
   [junit4]   2> 21973 T31 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 27085 T31 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=147469593
   [junit4]   2> 27090 T31 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> ASYNC  NEW_CORE C6 name=collection1 org.apache.solr.core.SolrCore@76ad079c
   [junit4]   2> 27206 T65 C6 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=10 status=0 QTime=391 
   [junit4]   2> ASYNC  NEW_CORE C7 name=collection1 org.apache.solr.core.SolrCore@701f8f5b
   [junit4]   2> 27207 T41 C7 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=10 status=0 QTime=672 
   [junit4]   2> 27269 T41 C7 oasc.SolrException.log ERROR null:org.eclipse.jetty.io.EofException
   [junit4]   2> 		at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:142)
   [junit4]   2> 		at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:107)
   [junit4]   2> 		at org.apache.solr.common.util.FastOutputStream.flush(FastOutputStream.java:214)
   [junit4]   2> 		at org.apache.solr.common.util.FastOutputStream.flushBuffer(FastOutputStream.java:207)
   [junit4]   2> 		at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:98)
   [junit4]   2> 		at org.apache.solr.response.BinaryResponseWriter.write(BinaryResponseWriter.java:52)
   [junit4]   2> 		at org.apache.solr.response.QueryResponseWriterUtil.writeQueryResponse(QueryResponseWriterUtil.java:50)
   [junit4]   2> 		at org.apache.solr.servlet.SolrDispatchFilter.writeResponse(SolrDispatchFilter.java:771)
   [junit4]   2> 		at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:424)
   [junit4]   2> 		at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]   2> 		at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:137)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
   [junit4]   2> 		at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
   [junit4]   2> 		at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
   [junit4]   2> 		at org.eclipse.jetty.server.Server.handle(Server.java:368)
   [junit4]   2> 		at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
   [junit4]   2> 		at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
   [junit4]   2> 		at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
   [junit4]   2> 		at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
   [junit4]   2> 		at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
   [junit4]   2> 		at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
   [junit4]   2> 		at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:628)
   [junit4]   2> 		at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
   [junit4]   2> 		at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]   2> 		at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]   2> 		at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	
   [junit4]   2> 27271 T41 C7 oasc.SolrException.log ERROR null:org.eclipse.jetty.io.EofException
   [junit4]   2> 		at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:142)
   [junit4]   2> 		at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:107)
   [junit4]   2> 		at org.apache.solr.common.util.FastOutputStream.flush(FastOutputStream.java:214)
   [junit4]   2> 		at org.apache.solr.common.util.FastOutputStream.flushBuffer(FastOutputStream.java:207)
   [junit4]   2> 		at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:98)
   [junit4]   2> 		at org.apache.solr.response.BinaryResponseWriter.write(BinaryResponseWriter.java:52)
   [junit4]   2> 		at org.apache.solr.response.QueryResponseWriterUtil.writeQueryResponse(QueryResponseWriterUtil.java:50)
   [junit4]   2> 		at org.apache.solr.servlet.SolrDispatchFilter.writeResponse(SolrDispatchFilter.java:771)
   [junit4]   2> 		at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:424)
   [junit4]   2> 		at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]   2> 		at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:137)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
   [junit4]   2> 		at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
   [junit4]   2> 		at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
   [junit4]   2> 		at org.eclipse.jetty.server.Server.handle(Server.java:368)
   [junit4]   2> 		at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
   [junit4]   2> 		at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
   [junit4]   2> 		at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
   [junit4]   2> 		at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
   [junit4]   2> 		at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
   [junit4]   2> 		at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
   [junit4]   2> 		at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:628)
   [junit4]   2> 		at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
   [junit4]   2> 		at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]   2> 		at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]   2> 		at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	
   [junit4]   2> 27297 T41 C7 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@701f8f5b
   [junit4]   2> 27298 T41 C7 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=10,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 27301 T41 C7 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 27302 T41 C7 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 27304 T41 C7 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 27343 T41 C7 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 27347 T41 C7 oasc.CachingDirectoryFactory.close Closing MockFSDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 27347 T41 C7 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data;done=false>>]
   [junit4]   2> 27348 T41 C7 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data
   [junit4]   2> 27350 T41 C7 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index;done=false>>]
   [junit4]   2> 27350 T41 C7 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index
   [junit4]   2> 27508 T61 C6 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={q=*:*&distrib=false&sort=_docid_+asc&rows=0&wt=javabin&version=2} hits=10 status=0 QTime=138 
   [junit4]   2> 32103 T31 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=153560465
   [junit4]   2> 32105 T31 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@76ad079c
   [junit4]   2> 32106 T31 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=10,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 32108 T31 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 32108 T31 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 32109 T31 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 32117 T31 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 32122 T31 oasc.CachingDirectoryFactory.close Closing MockFSDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 32122 T31 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data;done=false>>]
   [junit4]   2> 32122 T31 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data
   [junit4]   2> 32123 T31 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index;done=false>>]
   [junit4]   2> 32124 T31 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index
   [junit4]   2> 32159 T31 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 37107 T31 oas.SolrTestCaseJ4.tearDown ###Ending testReliability
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLBHttpSolrServer -Dtests.method=testReliability -Dtests.seed=2E5BBD83E82DC359 -Dtests.slow=true -Dtests.locale=nl_BE -Dtests.timezone=Poland -Dtests.file.encoding=UTF-8
   [junit4] ERROR   22.2s | TestLBHttpSolrServer.testReliability <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2E5BBD83E82DC359:EF9360C5494B12F0]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:528)
   [junit4]    > 	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
   [junit4]    > 	at org.apache.solr.client.solrj.TestLBHttpSolrServer.testReliability(TestLBHttpSolrServer.java:223)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:53542/solr
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:560)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:481)
   [junit4]    > 	... 43 more
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:150)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:121)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
   [junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:448)
   [junit4]    > 	... 46 more
   [junit4]   2> 37260 T31 oas.SolrTestCaseJ4.setUp ###Starting testTwoServers
   [junit4]   2> 37274 T31 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 37286 T31 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53588
   [junit4]   2> 37288 T31 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 37289 T31 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 37289 T31 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001
   [junit4]   2> 37290 T31 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/'
   [junit4]   2> 37372 T31 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/solr.xml
   [junit4]   2> 37436 T31 oasc.CoreContainer.<init> New CoreContainer 1435178801
   [junit4]   2> 37436 T31 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/]
   [junit4]   2> 37437 T31 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 37438 T31 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 37438 T31 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 37439 T31 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 37441 T31 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 37447 T31 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 37448 T31 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 37448 T31 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 37449 T31 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 37450 T31 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 37451 T31 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 37459 T31 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 37460 T31 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 37468 T80 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/'
   [junit4]   2> 37520 T80 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 37540 T80 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 37541 T80 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 37550 T80 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 37565 T80 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 37567 T80 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/
   [junit4]   2> 37567 T80 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockFSDirectoryFactory
   [junit4]   2> 37567 T80 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/, dataDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/
   [junit4]   2> 37567 T80 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 37575 T80 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data
   [junit4]   2> 37576 T80 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index/
   [junit4]   2> 37576 T80 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 37579 T80 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index
   [junit4]   2> 37585 T80 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 37585 T80 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 37589 T80 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 37590 T80 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 37590 T80 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 37590 T80 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 37591 T80 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 37591 T80 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 37591 T80 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 37591 T80 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 37591 T80 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 37592 T80 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 37601 T80 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37606 T80 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37613 T80 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37614 T80 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37616 T80 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 37617 T80 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 37623 T80 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 37623 T80 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 37625 T80 oass.SolrIndexSearcher.<init> Opening Searcher@53144aa5[collection1] main
   [junit4]   2> 37625 T80 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/conf
   [junit4]   2> 37626 T80 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/conf}
   [junit4]   2> 37626 T80 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/conf
   [junit4]   2> 37627 T80 oasr.ManagedResource.reloadFromStorage WARN No stored data found for /rest/managed
   [junit4]   2> 37628 T80 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using file:dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/conf
   [junit4]   2> 37629 T80 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 37629 T80 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 37630 T81 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@53144aa5[collection1] main{UninvertingDirectoryReader()}
   [junit4]   2> 37630 T80 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 37641 T31 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0
   [junit4]   2> 37642 T31 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> ASYNC  NEW_CORE C8 name=collection1 org.apache.solr.core.SolrCore@4a570d65
   [junit4]   2> 37678 T75 C8 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]} 0 29
   [junit4]   2> 37687 T76 C8 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 37733 T76 C8 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index),segFN=segments_2,generation=2}
   [junit4]   2> 37734 T76 C8 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 37754 T76 C8 oass.SolrIndexSearcher.<init> Opening Searcher@3e3eec63[collection1] main
   [junit4]   2> 37755 T76 C8 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 37755 T81 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3e3eec63[collection1] main{UninvertingDirectoryReader(Uninverting(_0(5.0.0):c10))}
   [junit4]   2> 37760 T76 C8 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} {commit=} 0 73
   [junit4]   2> 37775 T31 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 37785 T31 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53592
   [junit4]   2> 37786 T31 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 37786 T31 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 37786 T31 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001
   [junit4]   2> 37787 T31 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/'
   [junit4]   2> 37809 T31 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/solr.xml
   [junit4]   2> 37846 T31 oasc.CoreContainer.<init> New CoreContainer 1130257930
   [junit4]   2> 37849 T31 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/]
   [junit4]   2> 37850 T31 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 37850 T31 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 37850 T31 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 37851 T31 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 37851 T31 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 37852 T31 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 37853 T31 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 37853 T31 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 37853 T31 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 37854 T31 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 37854 T31 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 37854 T31 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 37855 T31 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 37866 T92 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/'
   [junit4]   2> 37905 T92 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 37930 T92 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 37932 T92 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 37937 T92 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 37945 T92 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 37947 T92 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/
   [junit4]   2> 37948 T92 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockFSDirectoryFactory
   [junit4]   2> 37948 T92 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/, dataDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/
   [junit4]   2> 37949 T92 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 37951 T92 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data
   [junit4]   2> 37952 T92 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index/
   [junit4]   2> 37952 T92 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 37954 T92 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index
   [junit4]   2> 37962 T92 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 37962 T92 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 37965 T92 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 37966 T92 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 37966 T92 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 37967 T92 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 37967 T92 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 37969 T92 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 37969 T92 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 37969 T92 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 37970 T92 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 37970 T92 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 37972 T92 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37973 T92 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37974 T92 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37976 T92 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37978 T92 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 37978 T92 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 37983 T92 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index lockFactory=NativeFSLockFactory@/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 37983 T92 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 37984 T92 oass.SolrIndexSearcher.<init> Opening Searcher@bf9669f[collection1] main
   [junit4]   2> 37984 T92 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/conf
   [junit4]   2> 37985 T92 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/conf}
   [junit4]   2> 37985 T92 oasr.Ma

[...truncated too long message...]

ve_adds=10,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 67616 T31 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 67617 T31 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 67617 T31 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 67624 T31 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 67625 T31 oasc.CachingDirectoryFactory.close Closing MockFSDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 67626 T31 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index;done=false>>]
   [junit4]   2> 67627 T31 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data/index
   [junit4]   2> 67643 T31 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data;done=false>>]
   [junit4]   2> 67644 T31 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-0-001/collection1/data
   [junit4]   2> 67651 T31 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 72628 T31 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=582649744
   [junit4]   2> 72632 T31 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@6907c2fd
   [junit4]   2> 72633 T31 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 72635 T31 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 72636 T31 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 72636 T31 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 72651 T31 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 72653 T31 oasc.CachingDirectoryFactory.close Closing MockFSDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 72653 T31 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data;done=false>>]
   [junit4]   2> 72654 T31 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data
   [junit4]   2> 72654 T31 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index;done=false>>]
   [junit4]   2> 72655 T31 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-1-001/collection1/data/index
   [junit4]   2> 72675 T31 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 77647 T31 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=657395488
   [junit4]   2> 77650 T31 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@2e900bea
   [junit4]   2> 77650 T31 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=10,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 77652 T31 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 77653 T31 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 77653 T31 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 77661 T31 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 77662 T31 oasc.CachingDirectoryFactory.close Closing MockFSDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 77663 T31 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data;done=false>>]
   [junit4]   2> 77663 T31 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data
   [junit4]   2> 77664 T31 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index [CachedDir<<refCount=0;path=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index;done=false>>]
   [junit4]   2> 77664 T31 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001/instance-2-001/collection1/data/index
   [junit4]   2> 77679 T31 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 82641 T31 oas.SolrTestCaseJ4.tearDown ###Ending testSimple
   [junit4]   2> 82643 T31 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.TestLBHttpSolrServer-2E5BBD83E82DC359-001
   [junit4]   2> NOTE: test params are: codec=Lucene410: {name=PostingsFormat(name=SimpleText), id=PostingsFormat(name=SimpleText)}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=nl_BE, timezone=Poland
   [junit4]   2> NOTE: Mac OS X 10.8.5 x86_64/Oracle Corporation 1.8.0_20 (64-bit)/cpus=2,threads=4,free=8372592,total=40632320
   [junit4]   2> NOTE: All tests run in this JVM: [DocumentAnalysisResponseTest, TestXMLEscaping, TestEmbeddedSolrServer, TestLBHttpSolrServer]
   [junit4] Completed in 67.66s, 3 tests, 1 error <<< FAILURES!

[...truncated 150 lines...]
BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:491: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:471: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:60: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/extra-targets.xml:39: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:202: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:496: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1358: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:961: There were test failures: 53 suites, 313 tests, 1 error

Total time: 158 minutes 12 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any