You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/06/22 15:52:07 UTC

[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.7.0) - Build # 586 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/586/
Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseG1GC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:50671/_w/i/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50671/_w/i/collection1
	at __randomizedtesting.SeedInfo.seed([1B9D7ED53A28CEB8:9A7BF0CD4D77AE84]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:435)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:90)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1049)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:137)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)
Caused by: java.net.SocketException: Invalid argument
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:150)
	at java.net.SocketInputStream.read(SocketInputStream.java:121)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:166)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:90)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:281)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:92)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:62)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:254)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:289)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:252)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:191)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:300)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:127)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:717)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:522)
	at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
	at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
	at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
	... 46 more




Build Log:
[...truncated 17594 lines...]
[junit4:junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
[junit4:junit4]   2> 1900977 T4431 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_w/i
[junit4:junit4]   2> 1900989 T4431 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4:junit4]   2> Creating dataDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./solrtest-ChaosMonkeySafeLeaderTest-1371907356854
[junit4:junit4]   2> 1900991 T4431 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4:junit4]   2> 1900992 T4432 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4:junit4]   2> 1901093 T4431 oasc.ZkTestServer.run start zk server on port:50615
[junit4:junit4]   2> 1901095 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1901109 T4438 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66032b76 name:ZooKeeperConnection Watcher:127.0.0.1:50615 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1901110 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1901112 T4431 oascc.SolrZkClient.makePath makePath: /solr
[junit4:junit4]   2> 1901127 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1901138 T4440 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ebe9d9b name:ZooKeeperConnection Watcher:127.0.0.1:50615/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1901138 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1901139 T4431 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4:junit4]   2> 1901149 T4431 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4:junit4]   2> 1901157 T4431 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4:junit4]   2> 1901168 T4431 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4:junit4]   2> 1901175 T4431 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4:junit4]   2> 1901181 T4431 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4:junit4]   2> 1901193 T4431 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4:junit4]   2> 1901197 T4431 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4:junit4]   2> 1901204 T4431 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4:junit4]   2> 1901205 T4431 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4:junit4]   2> 1901211 T4431 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4:junit4]   2> 1901212 T4431 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4:junit4]   2> 1901219 T4431 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4:junit4]   2> 1901220 T4431 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4:junit4]   2> 1901227 T4431 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4:junit4]   2> 1901228 T4431 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4:junit4]   2> 1901241 T4431 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4]   2> 1901243 T4431 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4]   2> 1901250 T4431 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4:junit4]   2> 1901251 T4431 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4:junit4]   2> 1901257 T4431 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4:junit4]   2> 1901258 T4431 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4:junit4]   2> 1901766 T4431 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4]   2> 1901777 T4431 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50618
[junit4:junit4]   2> 1901779 T4431 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4]   2> 1901780 T4431 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4]   2> 1901780 T4431 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371907357129
[junit4:junit4]   2> 1901781 T4431 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371907357129/solr.xml
[junit4:junit4]   2> 1901781 T4431 oasc.CoreContainer.<init> New CoreContainer 2014686949
[junit4:junit4]   2> 1901782 T4431 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371907357129/'
[junit4:junit4]   2> 1901783 T4431 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371907357129/'
[junit4:junit4]   2> 1901966 T4431 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4]   2> 1901967 T4431 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4]   2> 1901967 T4431 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4]   2> 1901968 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4]   2> 1901969 T4431 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4]   2> 1901969 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4]   2> 1901970 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4]   2> 1901970 T4431 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4]   2> 1901971 T4431 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4]   2> 1901971 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4]   2> 1901991 T4431 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4]   2> 1901992 T4431 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50615/solr
[junit4:junit4]   2> 1901992 T4431 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4]   2> 1901995 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1902000 T4451 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6695f08f name:ZooKeeperConnection Watcher:127.0.0.1:50615 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1902001 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1902005 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4]   2> 1902029 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1902033 T4453 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1108a17f name:ZooKeeperConnection Watcher:127.0.0.1:50615/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1902033 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1902036 T4431 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4:junit4]   2> 1902047 T4431 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4:junit4]   2> 1902096 T4431 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4:junit4]   2> 1902100 T4431 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50618__w%2Fi
[junit4:junit4]   2> 1902102 T4431 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50618__w%2Fi
[junit4:junit4]   2> 1902110 T4431 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4:junit4]   2> 1902131 T4431 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4:junit4]   2> 1902138 T4431 oasc.Overseer.start Overseer (id=89909320539570179-127.0.0.1:50618__w%2Fi-n_0000000000) starting
[junit4:junit4]   2> 1902162 T4431 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4:junit4]   2> 1902179 T4455 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4:junit4]   2> 1902181 T4454 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4:junit4]   2> 1902183 T4431 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4:junit4]   2> 1902189 T4431 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4:junit4]   2> 1902193 T4431 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
[junit4:junit4]   2> 1902206 T4456 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4]   2> 1902207 T4456 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1903714 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1903716 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"1",
[junit4:junit4]   2> 	  "numShards":"1",
[junit4:junit4]   2> 	  "shard":null,
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"down",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"control_collection",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50618__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50618/_w/i"}
[junit4:junit4]   2> 1903716 T4454 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
[junit4:junit4]   2> 1903717 T4454 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4]   2> 1903730 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4]   2> 1904211 T4456 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371907357129/collection1
[junit4:junit4]   2> 1904211 T4456 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4:junit4]   2> 1904221 T4456 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4]   2> 1904222 T4456 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4:junit4]   2> 1904226 T4456 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371907357129/collection1/'
[junit4:junit4]   2> 1904228 T4456 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371907357129/collection1/lib/classes/' to classloader
[junit4:junit4]   2> 1904228 T4456 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371907357129/collection1/lib/README' to classloader
[junit4:junit4]   2> 1904315 T4456 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4]   2> 1904409 T4456 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4]   2> 1904413 T4456 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4]   2> 1904421 T4456 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4]   2> 1905465 T4456 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4]   2> 1905484 T4456 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4]   2> 1905492 T4456 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4]   2> 1905544 T4456 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1905560 T4456 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1905587 T4456 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4]   2> 1905591 T4456 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4]   2> 1905591 T4456 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4]   2> 1905592 T4456 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4]   2> 1905596 T4456 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4]   2> 1905596 T4456 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4]   2> 1905597 T4456 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4]   2> 1905677 T4456 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371907357129/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/control/data/
[junit4:junit4]   2> 1905678 T4456 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@19582381
[junit4:junit4]   2> 1905678 T4456 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4]   2> 1905680 T4456 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/control/data
[junit4:junit4]   2> 1905681 T4456 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/control/data/index/
[junit4:junit4]   2> 1905681 T4456 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/control/data/index' doesn't exist. Creating new index...
[junit4:junit4]   2> 1905682 T4456 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/control/data/index
[junit4:junit4]   2> 1905690 T4456 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/control/data/index,segFN=segments_1,generation=1}
[junit4:junit4]   2> 1905690 T4456 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1905701 T4456 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4]   2> 1905702 T4456 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4]   2> 1905703 T4456 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4]   2> 1905704 T4456 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4]   2> 1905705 T4456 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4]   2> 1905705 T4456 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4]   2> 1905706 T4456 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4]   2> 1905707 T4456 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4]   2> 1905707 T4456 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4]   2> 1905723 T4456 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4]   2> 1905734 T4456 oass.SolrIndexSearcher.<init> Opening Searcher@5cd08c64 main
[junit4:junit4]   2> 1905737 T4456 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4]   2> 1905737 T4456 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4]   2> 1905744 T4457 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5cd08c64 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4]   2> 1905746 T4456 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4]   2> 1905747 T4456 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50618/_w/i collection:control_collection shard:shard1
[junit4:junit4]   2> 1905748 T4456 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4:junit4]   2> 1905765 T4456 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4]   2> 1905771 T4456 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4]   2> 1905771 T4456 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4]   2> 1905773 T4456 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50618/_w/i/collection1/
[junit4:junit4]   2> 1905776 T4456 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4]   2> 1905778 T4456 oasc.SyncStrategy.syncToMe http://127.0.0.1:50618/_w/i/collection1/ has no replicas
[junit4:junit4]   2> 1905778 T4456 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50618/_w/i/collection1/
[junit4:junit4]   2> 1905778 T4456 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4:junit4]   2> 1906755 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1906772 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4]   2> 1906833 T4456 oasc.ZkController.register We are http://127.0.0.1:50618/_w/i/collection1/ and leader is http://127.0.0.1:50618/_w/i/collection1/
[junit4:junit4]   2> 1906833 T4456 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50618/_w/i
[junit4:junit4]   2> 1906834 T4456 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4]   2> 1906834 T4456 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4]   2> 1906834 T4456 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1906838 T4456 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1906841 T4431 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4]   2> 1906841 T4431 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4]   2> 1906842 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1906863 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:
[junit4:junit4]   2> 1906872 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1906880 T4460 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@553e634e name:ZooKeeperConnection Watcher:127.0.0.1:50615/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1906880 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1906883 T4431 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
[junit4:junit4]   2> 1906890 T4431 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
[junit4:junit4]   2> 1907358 T4431 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4]   2> 1907365 T4431 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50622
[junit4:junit4]   2> 1907366 T4431 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4]   2> 1907367 T4431 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4]   2> 1907368 T4431 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371907362755
[junit4:junit4]   2> 1907368 T4431 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371907362755/solr.xml
[junit4:junit4]   2> 1907369 T4431 oasc.CoreContainer.<init> New CoreContainer 1791206176
[junit4:junit4]   2> 1907370 T4431 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371907362755/'
[junit4:junit4]   2> 1907370 T4431 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371907362755/'
[junit4:junit4]   2> 1907570 T4431 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4]   2> 1907571 T4431 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4]   2> 1907571 T4431 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4]   2> 1907572 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4]   2> 1907572 T4431 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4]   2> 1907573 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4]   2> 1907574 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4]   2> 1907574 T4431 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4]   2> 1907575 T4431 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4]   2> 1907575 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4]   2> 1907600 T4431 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4]   2> 1907602 T4431 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50615/solr
[junit4:junit4]   2> 1907602 T4431 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4]   2> 1907604 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1907609 T4471 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5cbbb120 name:ZooKeeperConnection Watcher:127.0.0.1:50615 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1907610 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1907614 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4]   2> 1907632 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1907637 T4473 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2aed7973 name:ZooKeeperConnection Watcher:127.0.0.1:50615/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1907638 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1907648 T4431 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
[junit4:junit4]   2> 1908287 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1908289 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"1",
[junit4:junit4]   2> 	  "numShards":"1",
[junit4:junit4]   2> 	  "shard":"shard1",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"active",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"control_collection",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50618__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50618/_w/i"}
[junit4:junit4]   2> 1908298 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4]   2> 1908298 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4]   2> 1908304 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4]   2> 1908656 T4431 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50622__w%2Fi
[junit4:junit4]   2> 1908659 T4431 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50622__w%2Fi
[junit4:junit4]   2> 1908671 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4]   2> 1908673 T4473 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4]   2> 1908673 T4460 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4]   2> 1908674 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1908674 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1908681 T4453 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4]   2> 1908692 T4474 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4]   2> 1908692 T4474 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1909813 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1909815 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"2",
[junit4:junit4]   2> 	  "numShards":"3",
[junit4:junit4]   2> 	  "shard":null,
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"down",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50622__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50622/_w/i"}
[junit4:junit4]   2> 1909815 T4454 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 3
[junit4:junit4]   2> 1909816 T4454 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4]   2> 1909824 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1909825 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1909826 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1910698 T4474 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371907362755/collection1
[junit4:junit4]   2> 1910699 T4474 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4]   2> 1910700 T4474 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4]   2> 1910701 T4474 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4]   2> 1910704 T4474 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371907362755/collection1/'
[junit4:junit4]   2> 1910706 T4474 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371907362755/collection1/lib/classes/' to classloader
[junit4:junit4]   2> 1910707 T4474 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371907362755/collection1/lib/README' to classloader
[junit4:junit4]   2> 1910800 T4474 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4]   2> 1910896 T4474 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4]   2> 1910899 T4474 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4]   2> 1910912 T4474 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4]   2> 1912106 T4474 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4]   2> 1912121 T4474 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4]   2> 1912127 T4474 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4]   2> 1912160 T4474 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1912174 T4474 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1912185 T4474 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4]   2> 1912189 T4474 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4]   2> 1912190 T4474 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4]   2> 1912190 T4474 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4]   2> 1912193 T4474 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4]   2> 1912193 T4474 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4]   2> 1912194 T4474 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4]   2> 1912195 T4474 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371907362755/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty1/
[junit4:junit4]   2> 1912195 T4474 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@19582381
[junit4:junit4]   2> 1912197 T4474 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4]   2> 1912198 T4474 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty1
[junit4:junit4]   2> 1912200 T4474 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty1/index/
[junit4:junit4]   2> 1912201 T4474 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty1/index' doesn't exist. Creating new index...
[junit4:junit4]   2> 1912202 T4474 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty1/index
[junit4:junit4]   2> 1912213 T4474 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty1/index,segFN=segments_1,generation=1}
[junit4:junit4]   2> 1912214 T4474 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1912222 T4474 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4]   2> 1912222 T4474 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4]   2> 1912224 T4474 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4]   2> 1912233 T4474 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4]   2> 1912235 T4474 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4]   2> 1912235 T4474 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4]   2> 1912237 T4474 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4]   2> 1912238 T4474 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4]   2> 1912239 T4474 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4]   2> 1912259 T4474 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4]   2> 1912270 T4474 oass.SolrIndexSearcher.<init> Opening Searcher@72a90538 main
[junit4:junit4]   2> 1912271 T4474 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4]   2> 1912272 T4474 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4]   2> 1912280 T4475 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@72a90538 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4]   2> 1912284 T4474 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4]   2> 1912284 T4474 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50622/_w/i collection:collection1 shard:shard1
[junit4:junit4]   2> 1912286 T4474 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4:junit4]   2> 1912315 T4474 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4]   2> 1912322 T4474 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4]   2> 1912323 T4474 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4]   2> 1912323 T4474 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50622/_w/i/collection1/
[junit4:junit4]   2> 1912323 T4474 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4]   2> 1912324 T4474 oasc.SyncStrategy.syncToMe http://127.0.0.1:50622/_w/i/collection1/ has no replicas
[junit4:junit4]   2> 1912324 T4474 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50622/_w/i/collection1/
[junit4:junit4]   2> 1912324 T4474 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4:junit4]   2> 1912847 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1912884 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1912885 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1912885 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1912916 T4474 oasc.ZkController.register We are http://127.0.0.1:50622/_w/i/collection1/ and leader is http://127.0.0.1:50622/_w/i/collection1/
[junit4:junit4]   2> 1912921 T4474 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50622/_w/i
[junit4:junit4]   2> 1912924 T4474 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4]   2> 1912925 T4474 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4]   2> 1912926 T4474 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1912936 T4474 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1912939 T4431 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4]   2> 1912941 T4431 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4]   2> 1912942 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1913642 T4431 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4]   2> 1913651 T4431 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50625
[junit4:junit4]   2> 1913655 T4431 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4]   2> 1913656 T4431 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4]   2> 1913657 T4431 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371907368836
[junit4:junit4]   2> 1913658 T4431 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371907368836/solr.xml
[junit4:junit4]   2> 1913659 T4431 oasc.CoreContainer.<init> New CoreContainer 250206443
[junit4:junit4]   2> 1913660 T4431 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371907368836/'
[junit4:junit4]   2> 1913661 T4431 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371907368836/'
[junit4:junit4]   2> 1913925 T4431 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4]   2> 1913927 T4431 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4]   2> 1913928 T4431 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4]   2> 1913928 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4]   2> 1913929 T4431 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4]   2> 1913931 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4]   2> 1913932 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4]   2> 1913933 T4431 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4]   2> 1913934 T4431 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4]   2> 1913935 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4]   2> 1913974 T4431 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4]   2> 1913975 T4431 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50615/solr
[junit4:junit4]   2> 1913976 T4431 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4]   2> 1913978 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1913989 T4487 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6441194c name:ZooKeeperConnection Watcher:127.0.0.1:50615 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1913991 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1913999 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4]   2> 1914041 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1914047 T4489 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@186e1599 name:ZooKeeperConnection Watcher:127.0.0.1:50615/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1914053 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1914071 T4431 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
[junit4:junit4]   2> 1914417 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1914427 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"2",
[junit4:junit4]   2> 	  "numShards":"3",
[junit4:junit4]   2> 	  "shard":"shard1",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"active",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50622__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50622/_w/i"}
[junit4:junit4]   2> 1914442 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1914443 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1914446 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1914447 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1915086 T4431 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50625__w%2Fi
[junit4:junit4]   2> 1915090 T4431 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50625__w%2Fi
[junit4:junit4]   2> 1915100 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1915107 T4460 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4]   2> 1915108 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1915108 T4489 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4]   2> 1915109 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1915110 T4473 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4]   2> 1915110 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1915124 T4453 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4]   2> 1915144 T4490 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4]   2> 1915147 T4490 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1915968 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1915971 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"3",
[junit4:junit4]   2> 	  "numShards":"3",
[junit4:junit4]   2> 	  "shard":null,
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"down",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50625__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50625/_w/i"}
[junit4:junit4]   2> 1915971 T4454 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4]   2> 1915972 T4454 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4]   2> 1916262 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1916263 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1916273 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1916274 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1917158 T4490 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371907368836/collection1
[junit4:junit4]   2> 1917159 T4490 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4]   2> 1917162 T4490 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4]   2> 1917162 T4490 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4]   2> 1917165 T4490 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371907368836/collection1/'
[junit4:junit4]   2> 1917167 T4490 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371907368836/collection1/lib/classes/' to classloader
[junit4:junit4]   2> 1917168 T4490 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371907368836/collection1/lib/README' to classloader
[junit4:junit4]   2> 1917288 T4490 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4]   2> 1917435 T4490 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4]   2> 1917439 T4490 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4]   2> 1917448 T4490 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4]   2> 1918893 T4490 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4]   2> 1918908 T4490 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4]   2> 1918913 T4490 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4]   2> 1918959 T4490 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1918972 T4490 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1918990 T4490 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4]   2> 1918996 T4490 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4]   2> 1918997 T4490 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4]   2> 1918998 T4490 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4]   2> 1919001 T4490 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4]   2> 1919002 T4490 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4]   2> 1919003 T4490 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4]   2> 1919004 T4490 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371907368836/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty2/
[junit4:junit4]   2> 1919004 T4490 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@19582381
[junit4:junit4]   2> 1919006 T4490 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4]   2> 1919008 T4490 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty2
[junit4:junit4]   2> 1919009 T4490 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty2/index/
[junit4:junit4]   2> 1919014 T4490 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty2/index' doesn't exist. Creating new index...
[junit4:junit4]   2> 1919019 T4490 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty2/index
[junit4:junit4]   2> 1919045 T4490 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty2/index,segFN=segments_1,generation=1}
[junit4:junit4]   2> 1919047 T4490 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1919055 T4490 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4]   2> 1919056 T4490 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4]   2> 1919059 T4490 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4]   2> 1919138 T4490 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4]   2> 1919140 T4490 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4]   2> 1919141 T4490 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4]   2> 1919143 T4490 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4]   2> 1919145 T4490 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4]   2> 1919146 T4490 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4]   2> 1919169 T4490 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4]   2> 1919187 T4490 oass.SolrIndexSearcher.<init> Opening Searcher@6c0772e2 main
[junit4:junit4]   2> 1919190 T4490 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4]   2> 1919191 T4490 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4]   2> 1919207 T4491 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6c0772e2 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4]   2> 1919217 T4490 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4]   2> 1919218 T4490 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50625/_w/i collection:collection1 shard:shard2
[junit4:junit4]   2> 1919220 T4490 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4:junit4]   2> 1919239 T4490 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4]   2> 1919246 T4490 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4]   2> 1919246 T4490 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4]   2> 1919247 T4490 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50625/_w/i/collection1/
[junit4:junit4]   2> 1919247 T4490 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4]   2> 1919248 T4490 oasc.SyncStrategy.syncToMe http://127.0.0.1:50625/_w/i/collection1/ has no replicas
[junit4:junit4]   2> 1919248 T4490 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50625/_w/i/collection1/
[junit4:junit4]   2> 1919248 T4490 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4:junit4]   2> 1919297 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1919322 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1919323 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1919324 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1919325 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1919337 T4490 oasc.ZkController.register We are http://127.0.0.1:50625/_w/i/collection1/ and leader is http://127.0.0.1:50625/_w/i/collection1/
[junit4:junit4]   2> 1919338 T4490 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50625/_w/i
[junit4:junit4]   2> 1919338 T4490 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4]   2> 1919339 T4490 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4]   2> 1919339 T4490 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1919351 T4490 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1919353 T4431 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4]   2> 1919354 T4431 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4]   2> 1919355 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1920015 T4431 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4]   2> 1920023 T4431 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50628
[junit4:junit4]   2> 1920026 T4431 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4]   2> 1920027 T4431 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4]   2> 1920027 T4431 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371907375250
[junit4:junit4]   2> 1920028 T4431 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371907375250/solr.xml
[junit4:junit4]   2> 1920028 T4431 oasc.CoreContainer.<init> New CoreContainer 265960198
[junit4:junit4]   2> 1920029 T4431 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371907375250/'
[junit4:junit4]   2> 1920030 T4431 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371907375250/'
[junit4:junit4]   2> 1920277 T4431 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4]   2> 1920278 T4431 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4]   2> 1920279 T4431 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4]   2> 1920279 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4]   2> 1920280 T4431 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4]   2> 1920281 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4]   2> 1920281 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4]   2> 1920282 T4431 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4]   2> 1920283 T4431 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4]   2> 1920283 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4]   2> 1920313 T4431 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4]   2> 1920315 T4431 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50615/solr
[junit4:junit4]   2> 1920316 T4431 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4]   2> 1920319 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1920339 T4503 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2e9f5620 name:ZooKeeperConnection Watcher:127.0.0.1:50615 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1920340 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1920346 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4]   2> 1920373 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1920381 T4505 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e740978 name:ZooKeeperConnection Watcher:127.0.0.1:50615/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1920381 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1920405 T4431 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
[junit4:junit4]   2> 1920851 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1920856 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"3",
[junit4:junit4]   2> 	  "numShards":"3",
[junit4:junit4]   2> 	  "shard":"shard2",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"active",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50625__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50625/_w/i"}
[junit4:junit4]   2> 1920878 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1920878 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1920878 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1920879 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1920879 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1921419 T4431 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50628__w%2Fi
[junit4:junit4]   2> 1921423 T4431 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50628__w%2Fi
[junit4:junit4]   2> 1921452 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1921468 T4505 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4]   2> 1921469 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1921469 T4460 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4]   2> 1921470 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1921470 T4489 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4]   2> 1921471 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1921471 T4473 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4]   2> 1921472 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1921475 T4453 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4]   2> 1921506 T4506 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4]   2> 1921507 T4506 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1922399 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1922401 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"4",
[junit4:junit4]   2> 	  "numShards":"3",
[junit4:junit4]   2> 	  "shard":null,
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"down",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50628__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50628/_w/i"}
[junit4:junit4]   2> 1922402 T4454 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4]   2> 1922402 T4454 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard3
[junit4:junit4]   2> 1922415 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1922421 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1922422 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1922423 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1922424 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1922511 T4506 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371907375250/collection1
[junit4:junit4]   2> 1922512 T4506 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4]   2> 1922515 T4506 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4]   2> 1922515 T4506 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4]   2> 1922519 T4506 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371907375250/collection1/'
[junit4:junit4]   2> 1922522 T4506 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371907375250/collection1/lib/classes/' to classloader
[junit4:junit4]   2> 1922523 T4506 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371907375250/collection1/lib/README' to classloader
[junit4:junit4]   2> 1922709 T4506 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4]   2> 1922867 T4506 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4]   2> 1922871 T4506 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4]   2> 1922884 T4506 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4]   2> 1924472 T4506 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4]   2> 1924496 T4506 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4]   2> 1924506 T4506 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4]   2> 1924565 T4506 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1924598 T4506 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1924613 T4506 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4]   2> 1924617 T4506 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4]   2> 1924618 T4506 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4]   2> 1924619 T4506 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4]   2> 1924623 T4506 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4]   2> 1924624 T4506 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4]   2> 1924624 T4506 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4]   2> 1924625 T4506 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371907375250/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty3/
[junit4:junit4]   2> 1924626 T4506 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@19582381
[junit4:junit4]   2> 1924627 T4506 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4]   2> 1924630 T4506 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty3
[junit4:junit4]   2> 1924631 T4506 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty3/index/
[junit4:junit4]   2> 1924634 T4506 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty3/index' doesn't exist. Creating new index...
[junit4:junit4]   2> 1924636 T4506 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty3/index
[junit4:junit4]   2> 1924647 T4506 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty3/index,segFN=segments_1,generation=1}
[junit4:junit4]   2> 1924648 T4506 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1924655 T4506 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4]   2> 1924656 T4506 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4]   2> 1924658 T4506 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4]   2> 1924660 T4506 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4]   2> 1924666 T4506 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4]   2> 1924669 T4506 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4]   2> 1924672 T4506 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4]   2> 1924674 T4506 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4]   2> 1924675 T4506 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4]   2> 1924699 T4506 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4]   2> 1924717 T4506 oass.SolrIndexSearcher.<init> Opening Searcher@24d69f96 main
[junit4:junit4]   2> 1924720 T4506 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4]   2> 1924721 T4506 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4]   2> 1924732 T4507 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@24d69f96 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4]   2> 1924739 T4506 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4]   2> 1924739 T4506 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50628/_w/i collection:collection1 shard:shard3
[junit4:junit4]   2> 1924745 T4506 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
[junit4:junit4]   2> 1924777 T4506 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4]   2> 1924786 T4506 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4]   2> 1924787 T4506 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4]   2> 1924787 T4506 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50628/_w/i/collection1/
[junit4:junit4]   2> 1924788 T4506 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4]   2> 1924789 T4506 oasc.SyncStrategy.syncToMe http://127.0.0.1:50628/_w/i/collection1/ has no replicas
[junit4:junit4]   2> 1924789 T4506 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50628/_w/i/collection1/
[junit4:junit4]   2> 1924790 T4506 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
[junit4:junit4]   2> 1925451 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1925476 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1925477 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1925477 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1925478 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1925479 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1925532 T4506 oasc.ZkController.register We are http://127.0.0.1:50628/_w/i/collection1/ and leader is http://127.0.0.1:50628/_w/i/collection1/
[junit4:junit4]   2> 1925533 T4506 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50628/_w/i
[junit4:junit4]   2> 1925533 T4506 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4]   2> 1925534 T4506 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4]   2> 1925534 T4506 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1925540 T4506 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1925544 T4431 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4]   2> 1925545 T4431 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4]   2> 1925546 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1926193 T4431 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4]   2> 1926202 T4431 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50631
[junit4:junit4]   2> 1926206 T4431 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4]   2> 1926207 T4431 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4]   2> 1926208 T4431 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371907381440
[junit4:junit4]   2> 1926208 T4431 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371907381440/solr.xml
[junit4:junit4]   2> 1926209 T4431 oasc.CoreContainer.<init> New CoreContainer 1341166329
[junit4:junit4]   2> 1926212 T4431 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371907381440/'
[junit4:junit4]   2> 1926213 T4431 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371907381440/'
[junit4:junit4]   2> 1926467 T4431 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4]   2> 1926468 T4431 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4]   2> 1926468 T4431 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4]   2> 1926468 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4]   2> 1926469 T4431 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4]   2> 1926469 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4]   2> 1926470 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4]   2> 1926470 T4431 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4]   2> 1926471 T4431 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4]   2> 1926471 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4]   2> 1926497 T4431 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4]   2> 1926503 T4431 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50615/solr
[junit4:junit4]   2> 1926506 T4431 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4]   2> 1926509 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1926519 T4519 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@390e6f name:ZooKeeperConnection Watcher:127.0.0.1:50615 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1926520 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1926543 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4]   2> 1926566 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1926575 T4521 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@56ba1c24 name:ZooKeeperConnection Watcher:127.0.0.1:50615/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1926578 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1926595 T4431 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
[junit4:junit4]   2> 1927005 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1927009 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"4",
[junit4:junit4]   2> 	  "numShards":"3",
[junit4:junit4]   2> 	  "shard":"shard3",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"active",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50628__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50628/_w/i"}
[junit4:junit4]   2> 1927022 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1927023 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1927023 T4521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1927024 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1927024 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1927024 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1927605 T4431 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50631__w%2Fi
[junit4:junit4]   2> 1927608 T4431 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50631__w%2Fi
[junit4:junit4]   2> 1927615 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1927620 T4460 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4]   2> 1927621 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1927621 T4473 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4]   2> 1927621 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1927631 T4453 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4]   2> 1927645 T4521 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4]   2> 1927646 T4521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1927646 T4489 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4]   2> 1927647 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1927645 T4505 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4]   2> 1927648 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1927660 T4522 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4]   2> 1927660 T4522 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1928550 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1928553 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"5",
[junit4:junit4]   2> 	  "numShards":"3",
[junit4:junit4]   2> 	  "shard":null,
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"down",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50631__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50631/_w/i"}
[junit4:junit4]   2> 1928555 T4454 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4]   2> 1928556 T4454 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4]   2> 1928569 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1928570 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1928571 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1928571 T4521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1928572 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1928573 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1928670 T4522 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371907381440/collection1
[junit4:junit4]   2> 1928670 T4522 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4]   2> 1928672 T4522 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4]   2> 1928672 T4522 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4]   2> 1928677 T4522 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371907381440/collection1/'
[junit4:junit4]   2> 1928679 T4522 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371907381440/collection1/lib/classes/' to classloader
[junit4:junit4]   2> 1928679 T4522 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371907381440/collection1/lib/README' to classloader
[junit4:junit4]   2> 1928799 T4522 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4]   2> 1928930 T4522 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4]   2> 1928934 T4522 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4]   2> 1928945 T4522 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4]   2> 1930564 T4522 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4]   2> 1930595 T4522 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4]   2> 1930606 T4522 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4]   2> 1930654 T4522 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1930666 T4522 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1930678 T4522 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4]   2> 1930683 T4522 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4]   2> 1930683 T4522 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4]   2> 1930684 T4522 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4]   2> 1930689 T4522 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4]   2> 1930690 T4522 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4]   2> 1930690 T4522 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4]   2> 1930691 T4522 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371907381440/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty4/
[junit4:junit4]   2> 1930692 T4522 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@19582381
[junit4:junit4]   2> 1930693 T4522 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4]   2> 1930695 T4522 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty4
[junit4:junit4]   2> 1930697 T4522 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty4/index/
[junit4:junit4]   2> 1930700 T4522 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty4/index' doesn't exist. Creating new index...
[junit4:junit4]   2> 1930701 T4522 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty4/index
[junit4:junit4]   2> 1930717 T4522 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty4/index,segFN=segments_1,generation=1}
[junit4:junit4]   2> 1930718 T4522 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1930726 T4522 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4]   2> 1930727 T4522 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4]   2> 1930729 T4522 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4]   2> 1930732 T4522 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4]   2> 1930733 T4522 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4]   2> 1930734 T4522 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4]   2> 1930735 T4522 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4]   2> 1930736 T4522 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4]   2> 1930737 T4522 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4]   2> 1930752 T4522 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4]   2> 1930763 T4522 oass.SolrIndexSearcher.<init> Opening Searcher@2d22fd6f main
[junit4:junit4]   2> 1930765 T4522 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4]   2> 1930765 T4522 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4]   2> 1930774 T4523 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2d22fd6f main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4]   2> 1930779 T4522 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4]   2> 1930780 T4522 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50631/_w/i collection:collection1 shard:shard1
[junit4:junit4]   2> 1930787 T4522 oasc.ZkController.register We are http://127.0.0.1:50631/_w/i/collection1/ and leader is http://127.0.0.1:50622/_w/i/collection1/
[junit4:junit4]   2> 1930787 T4522 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50631/_w/i
[junit4:junit4]   2> 1930788 T4522 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4]   2> 1930788 T4522 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4]   2> ASYNC  NEW_CORE C929 name=collection1 org.apache.solr.core.SolrCore@49cb979a url=http://127.0.0.1:50631/_w/i/collection1 node=127.0.0.1:50631__w%2Fi C929_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:50631__w%2Fi, base_url=http://127.0.0.1:50631/_w/i}
[junit4:junit4]   2> 1930790 T4524 C929 P50631 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
[junit4:junit4]   2> 1930790 T4522 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1930790 T4524 C929 P50631 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4]   2> 1930791 T4524 C929 P50631 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4]   2> 1930792 T4524 C929 P50631 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1930793 T4431 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4]   2> 1930794 T4431 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4]   2> 1930795 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1930817 T4524 C929 P50631 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1930855 T4468 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4]   2> 1931514 T4431 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4]   2> 1931540 T4431 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50635
[junit4:junit4]   2> 1931544 T4431 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4]   2> 1931547 T4431 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4]   2> 1931548 T4431 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371907386679
[junit4:junit4]   2> 1931549 T4431 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371907386679/solr.xml
[junit4:junit4]   2> 1931550 T4431 oasc.CoreContainer.<init> New CoreContainer 326200447
[junit4:junit4]   2> 1931552 T4431 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371907386679/'
[junit4:junit4]   2> 1931555 T4431 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371907386679/'
[junit4:junit4]   2> 1931624 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1931627 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"5",
[junit4:junit4]   2> 	  "numShards":"3",
[junit4:junit4]   2> 	  "shard":"shard1",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"recovering",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50631__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50631/_w/i"}
[junit4:junit4]   2> 1931645 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1931646 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1931647 T4521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1931648 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1931649 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1931646 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1931857 T4468 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4]   2> 1931858 T4468 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=5&state=recovering&nodeName=127.0.0.1:50631__w%252Fi&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1003 
[junit4:junit4]   2> 1931875 T4431 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4]   2> 1931879 T4431 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4]   2> 1931882 T4431 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4]   2> 1931883 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4]   2> 1931884 T4431 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4]   2> 1931885 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4]   2> 1931886 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4]   2> 1931887 T4431 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4]   2> 1931888 T4431 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4]   2> 1931889 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4]   2> 1931926 T4431 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4]   2> 1931928 T4431 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50615/solr
[junit4:junit4]   2> 1931929 T4431 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4]   2> 1931932 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1931940 T4536 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e1f40bb name:ZooKeeperConnection Watcher:127.0.0.1:50615 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1931941 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1931955 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4]   2> 1931982 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1931989 T4538 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5100ed31 name:ZooKeeperConnection Watcher:127.0.0.1:50615/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1931989 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1932003 T4431 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
[junit4:junit4]   2> 1933019 T4431 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50635__w%2Fi
[junit4:junit4]   2> 1933022 T4431 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50635__w%2Fi
[junit4:junit4]   2> 1933037 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1933041 T4505 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4]   2> 1933041 T4538 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4]   2> 1933042 T4460 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4]   2> 1933042 T4473 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4]   2> 1933041 T4489 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4]   2> 1933043 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1933041 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1933043 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1933043 T4521 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4]   2> 1933046 T4521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1933043 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1933053 T4453 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4]   2> 1933062 T4539 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4]   2> 1933063 T4539 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1933161 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1933165 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"6",
[junit4:junit4]   2> 	  "numShards":"3",
[junit4:junit4]   2> 	  "shard":null,
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"down",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50635__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50635/_w/i"}
[junit4:junit4]   2> 1933166 T4454 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4]   2> 1933166 T4454 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4]   2> 1933180 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1933181 T4521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1933182 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1933181 T4538 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1933181 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1933180 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1933180 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2>  C929_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:50631__w%2Fi, base_url=http://127.0.0.1:50631/_w/i}
[junit4:junit4]   2> 1933861 T4524 C929 P50631 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:50622/_w/i/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4]   2> 1933862 T4524 C929 P50631 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:50631/_w/i START replicas=[http://127.0.0.1:50622/_w/i/collection1/] nUpdates=100
[junit4:junit4]   2> 1933863 T4524 C929 P50631 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4]   2> 1933863 T4524 C929 P50631 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4]   2> 1933864 T4524 C929 P50631 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4]   2> 1933864 T4524 C929 P50631 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4]   2> 1933864 T4524 C929 P50631 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4]   2> 1933865 T4524 C929 P50631 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:50622/_w/i/collection1/. core=collection1
[junit4:junit4]   2> 1933865 T4524 C929 P50631 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> ASYNC  NEW_CORE C930 name=collection1 org.apache.solr.core.SolrCore@42871554 url=http://127.0.0.1:50622/_w/i/collection1 node=127.0.0.1:50622__w%2Fi C930_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50622__w%2Fi, base_url=http://127.0.0.1:50622/_w/i, leader=true}
[junit4:junit4]   2> 1933900 T4465 C930 P50622 oasc.SolrCore.execute [collection1] webapp=/_w/i path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0 
[junit4:junit4]   2> 1933913 T4466 C930 P50622 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4]   2> 1933922 T4466 C930 P50622 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty1/index,segFN=segments_1,generation=1}
[junit4:junit4]   2> 1933923 T4466 C930 P50622 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1933927 T4466 C930 P50622 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty1/index,segFN=segments_1,generation=1}
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty1/index,segFN=segments_2,generation=2}
[junit4:junit4]   2> 1933928 T4466 C930 P50622 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4]   2> 1933930 T4466 C930 P50622 oass.SolrIndexSearcher.<init> Opening Searcher@7e6727dd realtime
[junit4:junit4]   2> 1933931 T4466 C930 P50622 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4]   2> 1933931 T4466 C930 P50622 oasup.LogUpdateProcessor.finish [collection1] webapp=/_w/i path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 18
[junit4:junit4]   2> 1933933 T4524 C929 P50631 oash.ReplicationHandler.inform Commits will be reserved for  10000
[junit4:junit4]   2> 1933934 T4524 C929 P50631 oash.SnapPuller.<init>  No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4]   2> 1933940 T4463 C930 P50622 oash.ReplicationHandler.inform Commits will be reserved for  10000
[junit4:junit4]   2> 1933941 T4463 C930 P50622 oasc.SolrCore.execute [collection1] webapp=/_w/i path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=2 
[junit4:junit4]   2> 1933942 T4524 C929 P50631 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4]   2> 1933942 T4524 C929 P50631 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4]   2> 1933943 T4524 C929 P50631 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4]   2> 1933948 T4464 C930 P50622 oasc.SolrCore.execute [collection1] webapp=/_w/i path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=2 
[junit4:junit4]   2> 1933950 T4524 C929 P50631 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4]   2> 1933954 T4524 C929 P50631 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty4/index.20130622092309815
[junit4:junit4]   2> 1933955 T4524 C929 P50631 oash.SnapPuller.fetchLatestIndex Starting download to org.apache.lucene.store.NIOFSDirectory@/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty4/index.20130622092309815 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4740cf7a fullCopy=false
[junit4:junit4]   2> 1933962 T4468 C930 P50622 oasc.SolrCore.execute [collection1] webapp=/_w/i path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0 
[junit4:junit4]   2> 1933967 T4524 C929 P50631 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4]   2> 1933972 T4524 C929 P50631 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4]   2> 1933973 T4524 C929 P50631 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4]   2> 1933981 T4524 C929 P50631 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty4/index,segFN=segments_1,generation=1}
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty4/index,segFN=segments_2,generation=2}
[junit4:junit4]   2> 1933982 T4524 C929 P50631 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4]   2> 1933983 T4524 C929 P50631 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4]   2> 1933988 T4524 C929 P50631 oass.SolrIndexSearcher.<init> Opening Searcher@4789ad23 main
[junit4:junit4]   2> 1933991 T4523 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4789ad23 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4]   2> 1933992 T4524 C929 P50631 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty4/index.20130622092309815 [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty4/index.20130622092309815;done=true>>]
[junit4:junit4]   2> 1933993 T4524 C929 P50631 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty4/index.20130622092309815
[junit4:junit4]   2> 1933993 T4524 C929 P50631 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty4/index.20130622092309815
[junit4:junit4]   2> 1933995 T4524 C929 P50631 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4]   2> 1933996 T4524 C929 P50631 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4]   2> 1933996 T4524 C929 P50631 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4]   2> 1933996 T4524 C929 P50631 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1934000 T4524 C929 P50631 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4]   2> 1934067 T4539 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371907386679/collection1
[junit4:junit4]   2> 1934068 T4539 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4]   2> 1934071 T4539 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4]   2> 1934071 T4539 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4]   2> 1934076 T4539 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371907386679/collection1/'
[junit4:junit4]   2> 1934078 T4539 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371907386679/collection1/lib/classes/' to classloader
[junit4:junit4]   2> 1934079 T4539 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371907386679/collection1/lib/README' to classloader
[junit4:junit4]   2> 1934198 T4539 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4]   2> 1934291 T4539 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4]   2> 1934294 T4539 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4]   2> 1934301 T4539 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4]   2> 1934775 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1934778 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"5",
[junit4:junit4]   2> 	  "numShards":"3",
[junit4:junit4]   2> 	  "shard":"shard1",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"active",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50631__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50631/_w/i"}
[junit4:junit4]   2> 1934787 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1934788 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1934788 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1934788 T4521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1934789 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1934791 T4538 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1934791 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1935588 T4539 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4]   2> 1935612 T4539 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4]   2> 1935617 T4539 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4]   2> 1935656 T4539 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1935670 T4539 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1935678 T4539 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4]   2> 1935681 T4539 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4]   2> 1935681 T4539 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4]   2> 1935682 T4539 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4]   2> 1935684 T4539 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4]   2> 1935685 T4539 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4]   2> 1935685 T4539 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4]   2> 1935686 T4539 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371907386679/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty5/
[junit4:junit4]   2> 1935686 T4539 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@19582381
[junit4:junit4]   2> 1935687 T4539 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4]   2> 1935688 T4539 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty5
[junit4:junit4]   2> 1935689 T4539 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty5/index/
[junit4:junit4]   2> 1935690 T4539 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty5/index' doesn't exist. Creating new index...
[junit4:junit4]   2> 1935691 T4539 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty5/index
[junit4:junit4]   2> 1935698 T4539 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty5/index,segFN=segments_1,generation=1}
[junit4:junit4]   2> 1935698 T4539 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1935703 T4539 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4]   2> 1935704 T4539 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4]   2> 1935706 T4539 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4]   2> 1935708 T4539 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4]   2> 1935711 T4539 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4]   2> 1935711 T4539 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4]   2> 1935717 T4539 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4]   2> 1935720 T4539 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4]   2> 1935721 T4539 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4]   2> 1935744 T4539 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4]   2> 1935785 T4539 oass.SolrIndexSearcher.<init> Opening Searcher@6120ae91 main
[junit4:junit4]   2> 1935790 T4539 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4]   2> 1935791 T4539 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4]   2> 1935810 T4542 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6120ae91 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4]   2> 1935815 T4539 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4]   2> 1935815 T4539 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50635/_w/i collection:collection1 shard:shard2
[junit4:junit4]   2> 1935840 T4539 oasc.ZkController.register We are http://127.0.0.1:50635/_w/i/collection1/ and leader is http://127.0.0.1:50625/_w/i/collection1/
[junit4:junit4]   2> 1935840 T4539 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50635/_w/i
[junit4:junit4]   2> 1935841 T4539 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4]   2> 1935841 T4539 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4]   2> ASYNC  NEW_CORE C931 name=collection1 org.apache.solr.core.SolrCore@78feb9d9 url=http://127.0.0.1:50635/_w/i/collection1 node=127.0.0.1:50635__w%2Fi C931_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:50635__w%2Fi, base_url=http://127.0.0.1:50635/_w/i}
[junit4:junit4]   2> 1935842 T4543 C931 P50635 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
[junit4:junit4]   2> 1935843 T4543 C931 P50635 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4]   2> 1935844 T4543 C931 P50635 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4]   2> 1935844 T4543 C931 P50635 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1935846 T4539 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1935847 T4543 C931 P50635 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1935870 T4431 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4]   2> 1935871 T4431 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4]   2> 1935872 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1935892 T4481 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 6, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4]   2> 1936323 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1936326 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"6",
[junit4:junit4]   2> 	  "numShards":"3",
[junit4:junit4]   2> 	  "shard":"shard2",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"recovering",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50635__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50635/_w/i"}
[junit4:junit4]   2> 1936345 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1936347 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1936347 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1936349 T4538 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1936350 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1936351 T4521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1936352 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1936548 T4431 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4]   2> 1936562 T4431 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50642
[junit4:junit4]   2> 1936564 T4431 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4]   2> 1936564 T4431 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4]   2> 1936565 T4431 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371907391758
[junit4:junit4]   2> 1936566 T4431 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371907391758/solr.xml
[junit4:junit4]   2> 1936566 T4431 oasc.CoreContainer.<init> New CoreContainer 270391433
[junit4:junit4]   2> 1936567 T4431 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371907391758/'
[junit4:junit4]   2> 1936567 T4431 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371907391758/'
[junit4:junit4]   2> 1936844 T4431 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4]   2> 1936846 T4431 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4]   2> 1936847 T4431 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4]   2> 1936848 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4]   2> 1936849 T4431 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4]   2> 1936850 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4]   2> 1936851 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4]   2> 1936852 T4431 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4]   2> 1936854 T4431 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4]   2> 1936855 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4]   2> 1936893 T4481 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 6, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4]   2> 1936894 T4481 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=6&state=recovering&nodeName=127.0.0.1:50635__w%252Fi&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1002 
[junit4:junit4]   2> 1936897 T4431 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4]   2> 1936898 T4431 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50615/solr
[junit4:junit4]   2> 1936901 T4431 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4]   2> 1936905 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1936912 T4555 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@ffe7416 name:ZooKeeperConnection Watcher:127.0.0.1:50615 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1936918 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1936942 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4]   2> 1936973 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1936976 T4557 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7899615b name:ZooKeeperConnection Watcher:127.0.0.1:50615/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1936977 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1936987 T4431 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
[junit4:junit4]   2> 1938003 T4431 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50642__w%2Fi
[junit4:junit4]   2> 1938007 T4431 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50642__w%2Fi
[junit4:junit4]   2> 1938016 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4]   2> 1938029 T4557 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4]   2> 1938030 T4473 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4]   2> 1938030 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1938031 T4489 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4]   2> 1938031 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1938031 T4460 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4]   2> 1938032 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1938033 T4505 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4]   2> 1938033 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1938034 T4521 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4]   2> 1938034 T4521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1938050 T4453 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4]   2> 1938051 T4538 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4]   2> 1938051 T4538 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1938075 T4558 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4]   2> 1938076 T4558 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2>  C931_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:50635__w%2Fi, base_url=http://127.0.0.1:50635/_w/i}
[junit4:junit4]   2> 1938898 T4543 C931 P50635 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:50625/_w/i/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4]   2> 1938901 T4543 C931 P50635 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:50635/_w/i START replicas=[http://127.0.0.1:50625/_w/i/collection1/] nUpdates=100
[junit4:junit4]   2> 1938903 T4543 C931 P50635 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4]   2> 1938903 T4543 C931 P50635 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4]   2> 1938904 T4543 C931 P50635 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4]   2> 1938904 T4543 C931 P50635 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4]   2> 1938904 T4543 C931 P50635 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4]   2> 1938904 T4543 C931 P50635 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:50625/_w/i/collection1/. core=collection1
[junit4:junit4]   2> 1938905 T4543 C931 P50635 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> ASYNC  NEW_CORE C932 name=collection1 org.apache.solr.core.SolrCore@3541ad7f url=http://127.0.0.1:50625/_w/i/collection1 node=127.0.0.1:50625__w%2Fi C932_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50625__w%2Fi, base_url=http://127.0.0.1:50625/_w/i, leader=true}
[junit4:junit4]   2> 1938938 T4484 C932 P50625 oasc.SolrCore.execute [collection1] webapp=/_w/i path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0 
[junit4:junit4]   2> 1938947 T4485 C932 P50625 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4]   2> 1938957 T4485 C932 P50625 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty2/index,segFN=segments_1,generation=1}
[junit4:junit4]   2> 1938959 T4485 C932 P50625 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1938969 T4485 C932 P50625 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty2/index,segFN=segments_1,generation=1}
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty2/index,segFN=segments_2,generation=2}
[junit4:junit4]   2> 1938972 T4485 C932 P50625 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4]   2> 1938974 T4485 C932 P50625 oass.SolrIndexSearcher.<init> Opening Searcher@3bc269c5 realtime
[junit4:junit4]   2> 1938975 T4485 C932 P50625 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4]   2> 1938976 T4485 C932 P50625 oasup.LogUpdateProcessor.finish [collection1] webapp=/_w/i path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 29
[junit4:junit4]   2> 1938980 T4543 C931 P50635 oash.ReplicationHandler.inform Commits will be reserved for  10000
[junit4:junit4]   2> 1938986 T4543 C931 P50635 oash.SnapPuller.<init>  No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4]   2> 1938994 T4482 C932 P50625 oash.ReplicationHandler.inform Commits will be reserved for  10000
[junit4:junit4]   2> 1938995 T4482 C932 P50625 oasc.SolrCore.execute [collection1] webapp=/_w/i path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=2 
[junit4:junit4]   2> 1938997 T4543 C931 P50635 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4]   2> 1938998 T4543 C931 P50635 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4]   2> 1938998 T4543 C931 P50635 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4]   2> 1939011 T4483 C932 P50625 oasc.SolrCore.execute [collection1] webapp=/_w/i path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=1 
[junit4:junit4]   2> 1939013 T4543 C931 P50635 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4]   2> 1939016 T4543 C931 P50635 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty5/index.20130622092314877
[junit4:junit4]   2> 1939017 T4543 C931 P50635 oash.SnapPuller.fetchLatestIndex Starting download to org.apache.lucene.store.NIOFSDirectory@/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty5/index.20130622092314877 lockFactory=org.apache.lucene.store.NativeFSLockFactory@14676c02 fullCopy=false
[junit4:junit4]   2> 1939022 T4481 C932 P50625 oasc.SolrCore.execute [collection1] webapp=/_w/i path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0 
[junit4:junit4]   2> 1939026 T4543 C931 P50635 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4]   2> 1939029 T4543 C931 P50635 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4]   2> 1939029 T4543 C931 P50635 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4]   2> 1939036 T4543 C931 P50635 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty5/index,segFN=segments_1,generation=1}
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty5/index,segFN=segments_2,generation=2}
[junit4:junit4]   2> 1939038 T4543 C931 P50635 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4]   2> 1939039 T4543 C931 P50635 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4]   2> 1939040 T4543 C931 P50635 oass.SolrIndexSearcher.<init> Opening Searcher@9fb253d main
[junit4:junit4]   2> 1939041 T4542 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@9fb253d main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4]   2> 1939042 T4543 C931 P50635 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty5/index.20130622092314877 [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty5/index.20130622092314877;done=true>>]
[junit4:junit4]   2> 1939042 T4543 C931 P50635 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty5/index.20130622092314877
[junit4:junit4]   2> 1939043 T4543 C931 P50635 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty5/index.20130622092314877
[junit4:junit4]   2> 1939044 T4543 C931 P50635 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4]   2> 1939044 T4543 C931 P50635 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4]   2> 1939045 T4543 C931 P50635 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4]   2> 1939045 T4543 C931 P50635 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1939048 T4543 C931 P50635 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4]   2> 1939389 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1939392 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"7",
[junit4:junit4]   2> 	  "numShards":"3",
[junit4:junit4]   2> 	  "shard":null,
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"down",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50642__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50642/_w/i"}
[junit4:junit4]   2> 1939393 T4454 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4]   2> 1939393 T4454 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard3
[junit4:junit4]   2> 1939410 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"6",
[junit4:junit4]   2> 	  "numShards":"3",
[junit4:junit4]   2> 	  "shard":"shard2",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"active",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50635__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50635/_w/i"}
[junit4:junit4]   2> 1939428 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1939429 T4557 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1939429 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1939429 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1939430 T4521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1939430 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1939429 T4538 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1939429 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1940082 T4558 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371907391758/collection1
[junit4:junit4]   2> 1940082 T4558 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4]   2> 1940085 T4558 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4]   2> 1940085 T4558 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4]   2> 1940097 T4558 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371907391758/collection1/'
[junit4:junit4]   2> 1940100 T4558 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371907391758/collection1/lib/classes/' to classloader
[junit4:junit4]   2> 1940104 T4558 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371907391758/collection1/lib/README' to classloader
[junit4:junit4]   2> 1940423 T4558 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4]   2> 1940595 T4558 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4]   2> 1940599 T4558 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4]   2> 1940614 T4558 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4]   2> 1942114 T4558 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4]   2> 1942135 T4558 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4]   2> 1942141 T4558 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4]   2> 1942192 T4558 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1942200 T4558 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1942209 T4558 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4]   2> 1942212 T4558 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4]   2> 1942213 T4558 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4]   2> 1942213 T4558 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4]   2> 1942218 T4558 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4]   2> 1942218 T4558 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4]   2> 1942219 T4558 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4]   2> 1942221 T4558 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371907391758/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty6/
[junit4:junit4]   2> 1942221 T4558 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@19582381
[junit4:junit4]   2> 1942223 T4558 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4]   2> 1942224 T4558 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty6
[junit4:junit4]   2> 1942226 T4558 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty6/index/
[junit4:junit4]   2> 1942227 T4558 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty6/index' doesn't exist. Creating new index...
[junit4:junit4]   2> 1942228 T4558 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty6/index
[junit4:junit4]   2> 1942242 T4558 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4]   2> 		commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371907356853/jetty6/index,segFN=segments_1,generation=1}
[junit4:junit4]   2> 1942243 T4558 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1942247 T4558 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4]   2> 1942248 T4558 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4]   2> 1942250 T4558 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4]   2> 1942251 T4558 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4]   2> 1942254 T4558 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4]   2> 1942255 T4558 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4]   2> 1942263 T4558 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4]   2> 1942265 T4558 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4]   2> 1942266 T4558 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4]   2> 1942295 T4558 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4]   2> 1942313 T4558 oass.SolrIndexSearcher.<init> Opening Searcher@64e3c633 main
[junit4:junit4]   2> 1942315 T4558 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4]   2> 1942315 T4558 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4]   2> 1942322 T4561 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@64e3c633 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4]   2> 1942329 T4558 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4]   2> 1942330 T4558 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50642/_w/i collection:collection1 shard:shard3
[junit4:junit4]   2> 1942338 T4558 oasc.ZkController.register We are http://127.0.0.1:50642/_w/i/collection1/ and leader is http://127.0.0.1:50628/_w/i/collection1/
[junit4:junit4]   2> 1942338 T4558 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50642/_w/i
[junit4:junit4]   2> 1942339 T4558 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4]   2> 1942339 T4558 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4]   2> ASYNC  NEW_CORE C933 name=collection1 org.apache.solr.core.SolrCore@5ae513c1 url=http://127.0.0.1:50642/_w/i/collection1 node=127.0.0.1:50642__w%2Fi C933_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:50642__w%2Fi, base_url=http://127.0.0.1:50642/_w/i}
[junit4:junit4]   2> 1942340 T4562 C933 P50642 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
[junit4:junit4]   2> 1942341 T4562 C933 P50642 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4]   2> 1942342 T4562 C933 P50642 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4]   2> 1942342 T4562 C933 P50642 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1942344 T4558 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1942346 T4562 C933 P50642 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1942349 T4431 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4]   2> 1942351 T4431 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4]   2> 1942352 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1942374 T4496 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 7, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4]   2> 1942478 T4454 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1942480 T4454 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"7",
[junit4:junit4]   2> 	  "numShards":"3",
[junit4:junit4]   2> 	  "shard":"shard3",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"recovering",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50642__w%2Fi",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50642/_w/i"}
[junit4:junit4]   2> 1942510 T4557 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1942511 T4538 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1942511 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1942511 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1942512 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1942512 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1942513 T4521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1942513 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1943095 T4431 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4]   2> 1943103 T4431 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50649
[junit4:junit4]   2> 1943106 T4431 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4]   2> 1943107 T4431 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4]   2> 1943108 T4431 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371907398235
[junit4:junit4]   2> 1943109 T4431 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371907398235/solr.xml
[junit4:junit4]   2> 1943110 T4431 oasc.CoreContainer.<init> New CoreContainer 342839032
[junit4:junit4]   2> 1943112 T4431 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371907398235/'
[junit4:junit4]   2> 1943119 T4431 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371907398235/'
[junit4:junit4]   2> 1943309 T4431 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4]   2> 1943310 T4431 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4]   2> 1943310 T4431 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4]   2> 1943311 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4]   2> 1943311 T4431 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4]   2> 1943311 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4]   2> 1943312 T4431 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4]   2> 1943312 T4431 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4]   2> 1943312 T4431 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4]   2> 1943313 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4]   2> 1943338 T4431 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4]   2> 1943340 T4431 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50615/solr
[junit4:junit4]   2> 1943341 T4431 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4]   2> 1943347 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1943360 T4574 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18509953 name:ZooKeeperConnection Watcher:127.0.0.1:50615 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1943361 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1943372 T4431 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4]   2> 1943376 T4496 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 7, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4]   2> 1943377 T4496 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=7&state=recovering&nodeName=127.0.0.1:50642__w%252Fi&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1003 
[junit4:junit4]   2> 1943402 T4431 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1943407 T4576 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@662c8ed6 name:ZooKeeperConnection Watcher:127.0.0.1:50615/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1943408 T4431 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1943439 T4431 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
[junit4:junit4]   2> 1944453 T4431 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50649__w%2Fi
[junit4:junit4]   2> 1944455 T4431 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50649__w%2Fi
[junit4:junit4]   2> 1944465 T4576 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4]   2> 1944465 T4505 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4]   2> 1944466 T4505 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4]   2> 1944467 T4557 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1944467 T4453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4]   2> 1944482 T4538 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4]   2> 1944483 T4538 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4]   2> 1944483 T4473 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4]   2> 1944484 T4473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4]   2> 1944485 T4460 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4]   2> 1944485 T4460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4]   2> 1944486 T4521 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4]   2> 1944486 T4521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4]   2> 1944483 T4489 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4]   2> 1944488 T4489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4]   2> 1944505 T4557 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4]   2> 1944505 T4453 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4]   2> 1944521 T4577 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4]   2> 1944522 T4577 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2>  C933_STATE=coll:collection1 core:collection1 props:{shard=shard3, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:50642__w%2Fi, base_url=http://127.0.0.1:50642/_w/i}
[junit4:junit4]   2> 1945380 T4562 C933 P50642 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:50628/_w/i/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4]   2> 1945381 T4562 C933 P50642 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:50642/_w/i START replicas=[http://127.0.0.1:50628/_w/i/collection1/] nUpdates=100
[junit4:junit4]   2> 1945382 T4562 C933 P50642 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4]   2> 1945383 T4562 C933 P50642 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4]   2> 1945383 T4562 C933 P50642 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4]   2> 1945383 T4562 C933 P50642 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4]   2> 1945384 T4562 C933 P50642 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4]   2> 1945384 T4562 C933 P50642 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:50628/_w/i/collection1/. core=collection1
[junit4:junit4]   2> 1945385 T4562 C933 P50642 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> ASYNC  NEW_CORE C934 name=collection1 org.apache.solr.core.SolrCore@aaf9b27 url=http://127.0.0.1:50628/_w/i/collect

[...truncated too long message...]

 ERROR    229s | ChaosMonkeySafeLeaderTest.testDistribSearch <<<
[junit4:junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50671/_w/i/collection1
[junit4:junit4]    > 	at __randomizedtesting.SeedInfo.seed([1B9D7ED53A28CEB8:9A7BF0CD4D77AE84]:0)
[junit4:junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:435)
[junit4:junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
[junit4:junit4]    > 	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:90)
[junit4:junit4]    > 	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
[junit4:junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1049)
[junit4:junit4]    > 	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:137)
[junit4:junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
[junit4:junit4]    > 	at java.lang.Thread.run(Thread.java:724)
[junit4:junit4]    > Caused by: java.net.SocketException: Invalid argument
[junit4:junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
[junit4:junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:150)
[junit4:junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:121)
[junit4:junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:166)
[junit4:junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:90)
[junit4:junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:281)
[junit4:junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:92)
[junit4:junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:62)
[junit4:junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:254)
[junit4:junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:289)
[junit4:junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:252)
[junit4:junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:191)
[junit4:junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:300)
[junit4:junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:127)
[junit4:junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:717)
[junit4:junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:522)
[junit4:junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
[junit4:junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
[junit4:junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
[junit4:junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
[junit4:junit4]    > 	... 46 more
[junit4:junit4]   2> 2129521 T4431 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4:junit4]   2> 228553 T4430 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
[junit4:junit4]   2> 2129584 T4672 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4]   2> 2129585 T4672 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4]   2> 2129586 T4672 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4]   2> NOTE: test params are: codec=Lucene42: {range_facet_l=MockFixedIntBlock(blockSize=829), text=PostingsFormat(name=Direct), _version_=Pulsing41(freqCutoff=3 minBlockSize=74 maxBlockSize=162), rnd_b=Lucene41(blocksize=128), intDefault=Pulsing41(freqCutoff=3 minBlockSize=74 maxBlockSize=162), timestamp=MockFixedIntBlock(blockSize=829), id=Lucene41(blocksize=128), a_t=MockFixedIntBlock(blockSize=829), range_facet_sl=Lucene41(blocksize=128), range_facet_si=MockFixedIntBlock(blockSize=829), other_tl1=Pulsing41(freqCutoff=3 minBlockSize=74 maxBlockSize=162), multiDefault=MockFixedIntBlock(blockSize=829), a_si=Pulsing41(freqCutoff=3 minBlockSize=74 maxBlockSize=162)}, docValues:{timestamp=DocValuesFormat(name=Disk)}, sim=DefaultSimilarity, locale=de_LU, timezone=EST5EDT
[junit4:junit4]   2> NOTE: Mac OS X 10.8.4 x86_64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=2,threads=2,free=325851016,total=536870912
[junit4:junit4]   2> NOTE: All tests run in this JVM: [SolrCoreTest, DistanceFunctionTest, TestNumberUtils, SliceStateTest, HardAutoCommitTest, MultiTermTest, SampleTest, TestPseudoReturnFields, SpellingQueryConverterTest, SolrIndexConfigTest, TestBinaryField, TestSolrQueryParser, TestRemoteStreaming, FieldAnalysisRequestHandlerTest, OverseerTest, TestSchemaSimilarityResource, CacheHeaderTest, FastVectorHighlighterTest, OutputWriterTest, PathHierarchyTokenizerFactoryTest, SearchHandlerTest, TestSolrCoreProperties, TestRealTimeGet, CollectionsAPIDistributedZkTest, NoCacheHeaderTest, StandardRequestHandlerTest, TestStressReorder, SolrRequestParserTest, DocumentBuilderTest, SignatureUpdateProcessorFactoryTest, UpdateParamsTest, CopyFieldTest, TestSurroundQueryParser, TestCSVResponseWriter, TestSerializedLuceneMatchVersion, TestSolrQueryParserDefaultOperatorResource, TestSolrJ, TestIndexSearcher, TestFastLRUCache, TestPropInjectDefaults, UnloadDistributedZkTest, RequestHandlersTest, TimeZoneUtilsTest, QueryParsingTest, DisMaxRequestHandlerTest, TestSolrDeletionPolicy1, TestBM25SimilarityFactory, TestSchemaVersionResource, DocValuesMultiTest, ReturnFieldsTest, SuggesterFSTTest, DirectSolrSpellCheckerTest, ExternalFileFieldSortTest, TestMaxScoreQueryParser, TestPerFieldSimilarity, RequiredFieldsTest, QueryElevationComponentTest, TestFieldResource, TestPHPSerializedResponseWriter, SolrInfoMBeanTest, TestDocumentBuilder, TestSolrQueryParserResource, TestBadConfig, ClusterStateTest, TestRTGBase, UniqFieldsUpdateProcessorFactoryTest, TestValueSourceCache, TestLMJelinekMercerSimilarityFactory, SolrIndexSplitterTest, AliasIntegrationTest, DocValuesTest, NumericFieldsTest, DistributedQueryElevationComponentTest, ShowFileRequestHandlerTest, TestSuggestSpellingConverter, TestReplicationHandler, SyncSliceTest, BadIndexSchemaTest, BasicFunctionalityTest, DefaultValueUpdateProcessorTest, TestHashPartitioner, TestLFUCache, TestMergePolicyConfig, TestJmxMonitoredMap, TestSolrDeletionPolicy2, TestSweetSpotSimilarityFactory, TestExtendedDismaxParser, TermVectorComponentDistributedTest, RegexBoostProcessorTest, SuggesterWFSTTest, TestQuerySenderNoQuery, TestQueryUtils, ShardRoutingTest, ZkControllerTest, TestDynamicFieldResource, UpdateRequestProcessorFactoryTest, SliceStateUpdateTest, TestCSVLoader, TestFuzzyAnalyzedSuggestions, TestFunctionQuery, TestQuerySenderListener, XmlUpdateRequestHandlerTest, TestReversedWildcardFilterFactory, SolrPluginUtilsTest, DOMUtilTest, BasicDistributedZkTest, TestQueryTypes, TestFieldCollectionResource, TestSolr4Spatial, TestManagedSchema, SpellCheckCollatorTest, PrimUtilsTest, SpellCheckComponentTest, FieldMutatingUpdateProcessorTest, TestSolrXml, LeaderElectionIntegrationTest, TestFastWriter, TestCharFilters, TestCloudManagedSchemaAddField, TestConfig, TestStressVersions, XsltUpdateRequestHandlerTest, TestDistributedSearch, ClusterStateUpdateTest, TestStressRecovery, WordBreakSolrSpellCheckerTest, OverseerCollectionProcessorTest, TestAnalyzedSuggestions, OpenExchangeRatesOrgProviderTest, IndexSchemaRuntimeFieldTest, QueryEqualityTest, SystemInfoHandlerTest, TestSchemaNameResource, EchoParamsTest, TestLuceneMatchVersion, TestFaceting, TestUniqueKeyFieldResource, IndexReaderFactoryTest, TestPluginEnable, TestManagedSchemaFieldResource, LoggingHandlerTest, SuggesterTSTTest, SuggesterTest, TestDFRSimilarityFactory, CSVRequestHandlerTest, ChaosMonkeySafeLeaderTest]
[junit4:junit4] Completed in 228.86s, 1 test, 1 error <<< FAILURES!

[...truncated 468 lines...]
BUILD FAILED
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:386: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:366: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:39: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:181: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:437: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1243: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:887: There were test failures: 297 suites, 1253 tests, 1 error, 18 ignored (6 assumptions)

Total time: 107 minutes 34 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.7.0) - Build # 587 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/587/
Java: 64bit/jdk1.7.0 -XX:+UseCompressedOops -XX:+UseG1GC

2 tests failed.
REGRESSION:  org.apache.solr.cloud.ShardSplitTest.testDistribSearch

Error Message:
Server at http://127.0.0.1:50917 returned non ok status:500, message:Server Error

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:50917 returned non ok status:500, message:Server Error
	at __randomizedtesting.SeedInfo.seed([ED5D37BBB19952BE:6CBBB9A3C6C63282]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
	at org.apache.solr.cloud.ShardSplitTest.splitShard(ShardSplitTest.java:228)
	at org.apache.solr.cloud.ShardSplitTest.doTest(ShardSplitTest.java:150)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


REGRESSION:  org.apache.solr.cloud.SyncSliceTest.testDistribSearch

Error Message:
shard1 is not consistent.  Got 305 from http://127.0.0.1:51169/collection1lastClient and got 259 from http://127.0.0.1:51176/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 305 from http://127.0.0.1:51169/collection1lastClient and got 259 from http://127.0.0.1:51176/collection1
	at __randomizedtesting.SeedInfo.seed([ED5D37BBB19952BE:6CBBB9A3C6C63282]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1018)
	at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:238)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)




Build Log:
[...truncated 9517 lines...]
[junit4:junit4] Suite: org.apache.solr.cloud.ShardSplitTest
[junit4:junit4]   2> 1574660 T3497 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
[junit4:junit4]   2> 1574675 T3497 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4:junit4]   2> Creating dataDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./solrtest-ShardSplitTest-1371936056644
[junit4:junit4]   2> 1574678 T3497 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4:junit4]   2> 1574680 T3498 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4:junit4]   2> 1574780 T3497 oasc.ZkTestServer.run start zk server on port:50910
[junit4:junit4]   2> 1574791 T3497 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1574802 T3504 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c4378b4 name:ZooKeeperConnection Watcher:127.0.0.1:50910 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1574803 T3497 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1574804 T3497 oascc.SolrZkClient.makePath makePath: /solr
[junit4:junit4]   2> 1574815 T3497 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1574822 T3506 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6af31e19 name:ZooKeeperConnection Watcher:127.0.0.1:50910/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1574822 T3497 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1574823 T3497 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4:junit4]   2> 1574833 T3497 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4:junit4]   2> 1574842 T3497 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4:junit4]   2> 1574849 T3497 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4:junit4]   2> 1574859 T3497 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4:junit4]   2> 1574860 T3497 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4:junit4]   2> 1574880 T3497 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4:junit4]   2> 1574881 T3497 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4:junit4]   2> 1574892 T3497 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4:junit4]   2> 1574894 T3497 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4:junit4]   2> 1574903 T3497 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4:junit4]   2> 1574905 T3497 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4:junit4]   2> 1574919 T3497 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4:junit4]   2> 1574921 T3497 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4:junit4]   2> 1574930 T3497 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4:junit4]   2> 1574931 T3497 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4:junit4]   2> 1574944 T3497 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4]   2> 1574946 T3497 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4]   2> 1574952 T3497 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4:junit4]   2> 1574953 T3497 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4:junit4]   2> 1574961 T3497 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4:junit4]   2> 1574962 T3497 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4:junit4]   2> 1575432 T3497 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4]   2> 1575472 T3497 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50913
[junit4:junit4]   2> 1575473 T3497 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4]   2> 1575474 T3497 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4]   2> 1575479 T3497 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-controljetty-1371936056938
[junit4:junit4]   2> 1575480 T3497 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-controljetty-1371936056938/solr.xml
[junit4:junit4]   2> 1575481 T3497 oasc.CoreContainer.<init> New CoreContainer 700978093
[junit4:junit4]   2> 1575481 T3497 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-controljetty-1371936056938/'
[junit4:junit4]   2> 1575482 T3497 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-controljetty-1371936056938/'
[junit4:junit4]   2> 1575674 T3497 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4]   2> 1575676 T3497 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4]   2> 1575677 T3497 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4]   2> 1575678 T3497 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4]   2> 1575680 T3497 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4]   2> 1575681 T3497 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4]   2> 1575682 T3497 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4]   2> 1575683 T3497 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4]   2> 1575684 T3497 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4]   2> 1575685 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4]   2> 1575716 T3497 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4]   2> 1575719 T3497 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50910/solr
[junit4:junit4]   2> 1575720 T3497 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4]   2> 1575726 T3497 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1575729 T3517 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5daf5690 name:ZooKeeperConnection Watcher:127.0.0.1:50910 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1575729 T3497 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1575734 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4]   2> 1575758 T3497 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1575763 T3519 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fce8e75 name:ZooKeeperConnection Watcher:127.0.0.1:50910/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1575764 T3497 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1575768 T3497 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4:junit4]   2> 1575775 T3497 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4:junit4]   2> 1575782 T3497 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4:junit4]   2> 1575787 T3497 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50913_
[junit4:junit4]   2> 1575789 T3497 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50913_
[junit4:junit4]   2> 1575796 T3497 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4:junit4]   2> 1575826 T3497 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4:junit4]   2> 1575833 T3497 oasc.Overseer.start Overseer (id=89911201408876547-127.0.0.1:50913_-n_0000000000) starting
[junit4:junit4]   2> 1575840 T3497 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4:junit4]   2> 1575850 T3521 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4:junit4]   2> 1575852 T3520 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4:junit4]   2> 1575857 T3497 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4:junit4]   2> 1575863 T3497 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4:junit4]   2> 1575868 T3497 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
[junit4:junit4]   2> 1575883 T3522 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4]   2> 1575884 T3522 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1577361 T3520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1577363 T3520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"1",
[junit4:junit4]   2> 	  "numShards":"1",
[junit4:junit4]   2> 	  "shard":null,
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"down",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"control_collection",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50913_",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50913"}
[junit4:junit4]   2> 1577363 T3520 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
[junit4:junit4]   2> 1577363 T3520 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4]   2> 1577371 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4]   2> 1577892 T3522 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-controljetty-1371936056938/collection1
[junit4:junit4]   2> 1577892 T3522 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4:junit4]   2> 1577894 T3522 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4]   2> 1577894 T3522 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4:junit4]   2> 1577898 T3522 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-controljetty-1371936056938/collection1/'
[junit4:junit4]   2> 1577899 T3522 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-controljetty-1371936056938/collection1/lib/classes/' to classloader
[junit4:junit4]   2> 1577900 T3522 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-controljetty-1371936056938/collection1/lib/README' to classloader
[junit4:junit4]   2> 1578021 T3522 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4]   2> 1578151 T3522 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4]   2> 1578155 T3522 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4]   2> 1578161 T3522 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4]   2> 1579286 T3522 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4]   2> 1579298 T3522 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4]   2> 1579301 T3522 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1579309 T3522 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1579374 T3522 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4]   2> 1579375 T3522 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-controljetty-1371936056938/collection1/, dataDir=./org.apache.solr.cloud.ShardSplitTest-1371936056643/control/data/
[junit4:junit4]   2> 1579375 T3522 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@620533a5
[junit4:junit4]   2> 1579376 T3522 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4]   2> 1579377 T3522 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371936056643/control/data
[junit4:junit4]   2> 1579378 T3522 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ShardSplitTest-1371936056643/control/data/index/
[junit4:junit4]   2> 1579378 T3522 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ShardSplitTest-1371936056643/control/data/index' doesn't exist. Creating new index...
[junit4:junit4]   2> 1579379 T3522 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371936056643/control/data/index
[junit4:junit4]   2> 1579388 T3522 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@55dd8d7b lockFactory=org.apache.lucene.store.NativeFSLockFactory@27a9f213),segFN=segments_1,generation=1}
[junit4:junit4]   2> 1579389 T3522 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1579397 T3522 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4]   2> 1579398 T3522 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4]   2> 1579401 T3522 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4]   2> 1579403 T3522 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4]   2> 1579404 T3522 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4]   2> 1579405 T3522 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4]   2> 1579406 T3522 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4]   2> 1579407 T3522 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4]   2> 1579408 T3522 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4]   2> 1579430 T3522 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4]   2> 1579449 T3522 oass.SolrIndexSearcher.<init> Opening Searcher@3201cb2d main
[junit4:junit4]   2> 1579451 T3522 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ShardSplitTest-1371936056643/control/data/tlog
[junit4:junit4]   2> 1579454 T3522 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4]   2> 1579455 T3522 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4]   2> 1579463 T3523 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3201cb2d main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4]   2> 1579468 T3522 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4]   2> 1579468 T3522 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50913 collection:control_collection shard:shard1
[junit4:junit4]   2> 1579472 T3522 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4:junit4]   2> 1579493 T3522 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4]   2> 1579501 T3522 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4]   2> 1579502 T3522 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4]   2> 1579503 T3522 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50913/collection1/
[junit4:junit4]   2> 1579503 T3522 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4]   2> 1579504 T3522 oasc.SyncStrategy.syncToMe http://127.0.0.1:50913/collection1/ has no replicas
[junit4:junit4]   2> 1579504 T3522 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50913/collection1/
[junit4:junit4]   2> 1579505 T3522 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4:junit4]   2> 1580414 T3520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1580440 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4]   2> 1580461 T3522 oasc.ZkController.register We are http://127.0.0.1:50913/collection1/ and leader is http://127.0.0.1:50913/collection1/
[junit4:junit4]   2> 1580462 T3522 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50913
[junit4:junit4]   2> 1580462 T3522 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4]   2> 1580462 T3522 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4]   2> 1580463 T3522 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1580467 T3522 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1580470 T3497 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4]   2> 1580471 T3497 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4]   2> 1580472 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1580497 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:
[junit4:junit4]   2> 1580509 T3497 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1580512 T3526 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@142b1f6a name:ZooKeeperConnection Watcher:127.0.0.1:50910/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1580513 T3497 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1580523 T3497 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
[junit4:junit4]   2> 1580535 T3497 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
[junit4:junit4]   2> 1581086 T3497 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4]   2> 1581096 T3497 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50917
[junit4:junit4]   2> 1581097 T3497 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4]   2> 1581098 T3497 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4]   2> 1581098 T3497 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty1-1371936062503
[junit4:junit4]   2> 1581099 T3497 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty1-1371936062503/solr.xml
[junit4:junit4]   2> 1581099 T3497 oasc.CoreContainer.<init> New CoreContainer 1100866948
[junit4:junit4]   2> 1581100 T3497 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty1-1371936062503/'
[junit4:junit4]   2> 1581101 T3497 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty1-1371936062503/'
[junit4:junit4]   2> 1581316 T3497 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4]   2> 1581322 T3497 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4]   2> 1581323 T3497 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4]   2> 1581324 T3497 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4]   2> 1581324 T3497 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4]   2> 1581325 T3497 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4]   2> 1581325 T3497 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4]   2> 1581326 T3497 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4]   2> 1581327 T3497 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4]   2> 1581328 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4]   2> 1581348 T3497 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4]   2> 1581349 T3497 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50910/solr
[junit4:junit4]   2> 1581350 T3497 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4]   2> 1581352 T3497 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1581357 T3537 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@212187d4 name:ZooKeeperConnection Watcher:127.0.0.1:50910 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1581358 T3497 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1581363 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4]   2> 1581381 T3497 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1581391 T3539 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@617b3521 name:ZooKeeperConnection Watcher:127.0.0.1:50910/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1581392 T3497 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1581403 T3497 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
[junit4:junit4]   2> 1581958 T3520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1581961 T3520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"1",
[junit4:junit4]   2> 	  "numShards":"1",
[junit4:junit4]   2> 	  "shard":"shard1",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"active",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"control_collection",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50913_",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50913"}
[junit4:junit4]   2> 1581978 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4]   2> 1581979 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4]   2> 1581979 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4]   2> 1582415 T3497 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50917_
[junit4:junit4]   2> 1582419 T3497 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50917_
[junit4:junit4]   2> 1582427 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4]   2> 1582428 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4]   2> 1582433 T3539 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4]   2> 1582433 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1582443 T3519 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4]   2> 1582444 T3526 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4]   2> 1582470 T3540 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4]   2> 1582474 T3540 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1583506 T3520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1583508 T3520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"2",
[junit4:junit4]   2> 	  "numShards":"2",
[junit4:junit4]   2> 	  "shard":null,
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"down",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50917_",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50917"}
[junit4:junit4]   2> 1583509 T3520 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 2
[junit4:junit4]   2> 1583510 T3520 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4]   2> 1583523 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1583524 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1583525 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1584481 T3540 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty1-1371936062503/collection1
[junit4:junit4]   2> 1584482 T3540 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4]   2> 1584484 T3540 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4]   2> 1584484 T3540 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4]   2> 1584488 T3540 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty1-1371936062503/collection1/'
[junit4:junit4]   2> 1584492 T3540 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty1-1371936062503/collection1/lib/classes/' to classloader
[junit4:junit4]   2> 1584493 T3540 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty1-1371936062503/collection1/lib/README' to classloader
[junit4:junit4]   2> 1584630 T3540 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4]   2> 1584771 T3540 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4]   2> 1584776 T3540 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4]   2> 1584787 T3540 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4]   2> 1586236 T3540 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4]   2> 1586243 T3540 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4]   2> 1586248 T3540 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1586271 T3540 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1586374 T3540 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4]   2> 1586376 T3540 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty1-1371936062503/collection1/, dataDir=./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty1/
[junit4:junit4]   2> 1586376 T3540 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@620533a5
[junit4:junit4]   2> 1586377 T3540 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4]   2> 1586379 T3540 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty1
[junit4:junit4]   2> 1586380 T3540 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty1/index/
[junit4:junit4]   2> 1586380 T3540 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty1/index' doesn't exist. Creating new index...
[junit4:junit4]   2> 1586382 T3540 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty1/index
[junit4:junit4]   2> 1586389 T3540 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@6a946aac lockFactory=org.apache.lucene.store.NativeFSLockFactory@20f63bea),segFN=segments_1,generation=1}
[junit4:junit4]   2> 1586390 T3540 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1586402 T3540 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4]   2> 1586404 T3540 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4]   2> 1586407 T3540 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4]   2> 1586410 T3540 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4]   2> 1586412 T3540 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4]   2> 1586413 T3540 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4]   2> 1586415 T3540 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4]   2> 1586418 T3540 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4]   2> 1586420 T3540 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4]   2> 1586449 T3540 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4]   2> 1586470 T3540 oass.SolrIndexSearcher.<init> Opening Searcher@5da5cccf main
[junit4:junit4]   2> 1586471 T3540 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty1/tlog
[junit4:junit4]   2> 1586472 T3540 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4]   2> 1586473 T3540 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4]   2> 1586483 T3541 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5da5cccf main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4]   2> 1586485 T3540 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4]   2> 1586485 T3540 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50917 collection:collection1 shard:shard1
[junit4:junit4]   2> 1586487 T3540 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4:junit4]   2> 1586507 T3540 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4]   2> 1586516 T3540 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4]   2> 1586517 T3540 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4]   2> 1586518 T3540 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50917/collection1/
[junit4:junit4]   2> 1586518 T3540 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4]   2> 1586519 T3540 oasc.SyncStrategy.syncToMe http://127.0.0.1:50917/collection1/ has no replicas
[junit4:junit4]   2> 1586520 T3540 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50917/collection1/
[junit4:junit4]   2> 1586520 T3540 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4:junit4]   2> 1586552 T3520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1586584 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1586585 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1586584 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1586594 T3540 oasc.ZkController.register We are http://127.0.0.1:50917/collection1/ and leader is http://127.0.0.1:50917/collection1/
[junit4:junit4]   2> 1586596 T3540 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50917
[junit4:junit4]   2> 1586596 T3540 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4]   2> 1586597 T3540 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4]   2> 1586598 T3540 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1586604 T3540 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1586610 T3497 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4]   2> 1586612 T3497 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4]   2> 1586613 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1587166 T3497 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4]   2> 1587195 T3497 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50920
[junit4:junit4]   2> 1587196 T3497 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4]   2> 1587197 T3497 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4]   2> 1587198 T3497 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty2-1371936068612
[junit4:junit4]   2> 1587198 T3497 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty2-1371936068612/solr.xml
[junit4:junit4]   2> 1587199 T3497 oasc.CoreContainer.<init> New CoreContainer 1780329248
[junit4:junit4]   2> 1587200 T3497 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty2-1371936068612/'
[junit4:junit4]   2> 1587200 T3497 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty2-1371936068612/'
[junit4:junit4]   2> 1587455 T3497 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4]   2> 1587456 T3497 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4]   2> 1587458 T3497 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4]   2> 1587459 T3497 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4]   2> 1587461 T3497 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4]   2> 1587462 T3497 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4]   2> 1587463 T3497 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4]   2> 1587464 T3497 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4]   2> 1587465 T3497 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4]   2> 1587466 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4]   2> 1587501 T3497 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4]   2> 1587502 T3497 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50910/solr
[junit4:junit4]   2> 1587504 T3497 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4]   2> 1587508 T3497 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1587515 T3553 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c3aa588 name:ZooKeeperConnection Watcher:127.0.0.1:50910 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1587520 T3497 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1587526 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4]   2> 1587547 T3497 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1587559 T3555 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66f6a4c name:ZooKeeperConnection Watcher:127.0.0.1:50910/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1587562 T3497 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1587575 T3497 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
[junit4:junit4]   2> 1588119 T3520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1588122 T3520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"2",
[junit4:junit4]   2> 	  "numShards":"2",
[junit4:junit4]   2> 	  "shard":"shard1",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"active",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50917_",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50917"}
[junit4:junit4]   2> 1588136 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1588137 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1588138 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1588140 T3555 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1588586 T3497 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50920_
[junit4:junit4]   2> 1588589 T3497 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50920_
[junit4:junit4]   2> 1588596 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1588596 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4]   2> 1588604 T3539 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4]   2> 1588605 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1588607 T3555 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4]   2> 1588608 T3555 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1588613 T3526 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4]   2> 1588616 T3519 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4]   2> 1588634 T3556 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4]   2> 1588635 T3556 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1589659 T3520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1589662 T3520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"3",
[junit4:junit4]   2> 	  "numShards":"2",
[junit4:junit4]   2> 	  "shard":null,
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"down",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50920_",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50920"}
[junit4:junit4]   2> 1589663 T3520 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4]   2> 1589663 T3520 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4]   2> 1589676 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1589682 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1589682 T3555 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1589683 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1590643 T3556 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty2-1371936068612/collection1
[junit4:junit4]   2> 1590644 T3556 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4]   2> 1590646 T3556 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4]   2> 1590647 T3556 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4]   2> 1590652 T3556 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty2-1371936068612/collection1/'
[junit4:junit4]   2> 1590655 T3556 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty2-1371936068612/collection1/lib/classes/' to classloader
[junit4:junit4]   2> 1590656 T3556 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty2-1371936068612/collection1/lib/README' to classloader
[junit4:junit4]   2> 1590772 T3556 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4]   2> 1591034 T3556 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4]   2> 1591038 T3556 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4]   2> 1591052 T3556 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4]   2> 1592239 T3556 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4]   2> 1592251 T3556 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4]   2> 1592257 T3556 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1592267 T3556 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1592344 T3556 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4]   2> 1592345 T3556 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty2-1371936068612/collection1/, dataDir=./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty2/
[junit4:junit4]   2> 1592345 T3556 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@620533a5
[junit4:junit4]   2> 1592346 T3556 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4]   2> 1592348 T3556 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty2
[junit4:junit4]   2> 1592348 T3556 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty2/index/
[junit4:junit4]   2> 1592349 T3556 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty2/index' doesn't exist. Creating new index...
[junit4:junit4]   2> 1592350 T3556 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty2/index
[junit4:junit4]   2> 1592356 T3556 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5675db24 lockFactory=org.apache.lucene.store.NativeFSLockFactory@6133671f),segFN=segments_1,generation=1}
[junit4:junit4]   2> 1592356 T3556 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1592365 T3556 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4]   2> 1592366 T3556 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4]   2> 1592369 T3556 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4]   2> 1592371 T3556 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4]   2> 1592372 T3556 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4]   2> 1592372 T3556 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4]   2> 1592373 T3556 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4]   2> 1592375 T3556 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4]   2> 1592376 T3556 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4]   2> 1592400 T3556 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4]   2> 1592415 T3556 oass.SolrIndexSearcher.<init> Opening Searcher@2de8a5c9 main
[junit4:junit4]   2> 1592416 T3556 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty2/tlog
[junit4:junit4]   2> 1592417 T3556 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4]   2> 1592418 T3556 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4]   2> 1592426 T3557 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2de8a5c9 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4]   2> 1592431 T3556 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4]   2> 1592432 T3556 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50920 collection:collection1 shard:shard2
[junit4:junit4]   2> 1592433 T3556 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4:junit4]   2> 1592451 T3556 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4]   2> 1592458 T3556 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4]   2> 1592459 T3556 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4]   2> 1592459 T3556 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50920/collection1/
[junit4:junit4]   2> 1592460 T3556 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4]   2> 1592460 T3556 oasc.SyncStrategy.syncToMe http://127.0.0.1:50920/collection1/ has no replicas
[junit4:junit4]   2> 1592460 T3556 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50920/collection1/
[junit4:junit4]   2> 1592461 T3556 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4:junit4]   2> 1592722 T3520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1592742 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1592742 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1592742 T3555 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1592742 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1592787 T3556 oasc.ZkController.register We are http://127.0.0.1:50920/collection1/ and leader is http://127.0.0.1:50920/collection1/
[junit4:junit4]   2> 1592788 T3556 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50920
[junit4:junit4]   2> 1592788 T3556 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4]   2> 1592789 T3556 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4]   2> 1592789 T3556 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1592793 T3556 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1592796 T3497 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4]   2> 1592798 T3497 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4]   2> 1592798 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1593382 T3497 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4]   2> 1593397 T3497 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50923
[junit4:junit4]   2> 1593405 T3497 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4]   2> 1593406 T3497 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4]   2> 1593407 T3497 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty3-1371936074787
[junit4:junit4]   2> 1593408 T3497 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty3-1371936074787/solr.xml
[junit4:junit4]   2> 1593409 T3497 oasc.CoreContainer.<init> New CoreContainer 1413083806
[junit4:junit4]   2> 1593410 T3497 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty3-1371936074787/'
[junit4:junit4]   2> 1593412 T3497 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty3-1371936074787/'
[junit4:junit4]   2> 1593656 T3497 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4]   2> 1593657 T3497 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4]   2> 1593658 T3497 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4]   2> 1593659 T3497 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4]   2> 1593661 T3497 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4]   2> 1593661 T3497 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4]   2> 1593662 T3497 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4]   2> 1593663 T3497 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4]   2> 1593663 T3497 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4]   2> 1593664 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4]   2> 1593684 T3497 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4]   2> 1593685 T3497 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50910/solr
[junit4:junit4]   2> 1593686 T3497 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4]   2> 1593690 T3497 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1593694 T3569 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b0b8cfd name:ZooKeeperConnection Watcher:127.0.0.1:50910 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1593695 T3497 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1593702 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4]   2> 1593730 T3497 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1593736 T3571 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7faf206a name:ZooKeeperConnection Watcher:127.0.0.1:50910/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1593737 T3497 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1593751 T3497 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
[junit4:junit4]   2> 1594263 T3520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1594267 T3520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"3",
[junit4:junit4]   2> 	  "numShards":"2",
[junit4:junit4]   2> 	  "shard":"shard2",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"active",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50920_",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50920"}
[junit4:junit4]   2> 1594286 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1594286 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1594286 T3555 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1594287 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1594287 T3571 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1594768 T3497 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50923_
[junit4:junit4]   2> 1594772 T3497 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50923_
[junit4:junit4]   2> 1594780 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1594781 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4]   2> 1594784 T3571 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4]   2> 1594785 T3571 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1594786 T3555 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4]   2> 1594787 T3555 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1594788 T3539 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4]   2> 1594789 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1594793 T3526 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4]   2> 1594796 T3519 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4]   2> 1594818 T3572 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4]   2> 1594818 T3572 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1595804 T3520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1595807 T3520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"4",
[junit4:junit4]   2> 	  "numShards":"2",
[junit4:junit4]   2> 	  "shard":null,
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"down",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50923_",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50923"}
[junit4:junit4]   2> 1595808 T3520 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4]   2> 1595808 T3520 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4]   2> 1595831 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1595833 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1595833 T3571 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1595834 T3555 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1595833 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1596824 T3572 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty3-1371936074787/collection1
[junit4:junit4]   2> 1596825 T3572 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4]   2> 1596826 T3572 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4]   2> 1596827 T3572 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4]   2> 1596830 T3572 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty3-1371936074787/collection1/'
[junit4:junit4]   2> 1596831 T3572 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty3-1371936074787/collection1/lib/classes/' to classloader
[junit4:junit4]   2> 1596832 T3572 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty3-1371936074787/collection1/lib/README' to classloader
[junit4:junit4]   2> 1596931 T3572 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4]   2> 1597015 T3572 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4]   2> 1597018 T3572 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4]   2> 1597029 T3572 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4]   2> 1598220 T3572 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4]   2> 1598224 T3572 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4]   2> 1598229 T3572 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1598240 T3572 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1598316 T3572 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4]   2> 1598317 T3572 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty3-1371936074787/collection1/, dataDir=./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty3/
[junit4:junit4]   2> 1598318 T3572 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@620533a5
[junit4:junit4]   2> 1598319 T3572 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4]   2> 1598322 T3572 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty3
[junit4:junit4]   2> 1598322 T3572 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty3/index/
[junit4:junit4]   2> 1598323 T3572 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty3/index' doesn't exist. Creating new index...
[junit4:junit4]   2> 1598324 T3572 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty3/index
[junit4:junit4]   2> 1598329 T3572 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@227155c3 lockFactory=org.apache.lucene.store.NativeFSLockFactory@601c2328),segFN=segments_1,generation=1}
[junit4:junit4]   2> 1598330 T3572 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1598337 T3572 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4]   2> 1598338 T3572 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4]   2> 1598340 T3572 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4]   2> 1598342 T3572 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4]   2> 1598347 T3572 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4]   2> 1598349 T3572 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4]   2> 1598350 T3572 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4]   2> 1598353 T3572 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4]   2> 1598355 T3572 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4]   2> 1598382 T3572 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4]   2> 1598393 T3572 oass.SolrIndexSearcher.<init> Opening Searcher@4b1c1dce main
[junit4:junit4]   2> 1598395 T3572 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty3/tlog
[junit4:junit4]   2> 1598396 T3572 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4]   2> 1598397 T3572 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4]   2> 1598413 T3573 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4b1c1dce main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4]   2> 1598418 T3572 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4]   2> 1598419 T3572 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50923 collection:collection1 shard:shard1
[junit4:junit4]   2> 1598428 T3572 oasc.ZkController.register We are http://127.0.0.1:50923/collection1/ and leader is http://127.0.0.1:50917/collection1/
[junit4:junit4]   2> 1598429 T3572 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50923
[junit4:junit4]   2> 1598429 T3572 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4]   2> 1598430 T3572 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4]   2> ASYNC  NEW_CORE C617 name=collection1 org.apache.solr.core.SolrCore@4328c4c0 url=http://127.0.0.1:50923/collection1 node=127.0.0.1:50923_ C617_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:50923_, base_url=http://127.0.0.1:50923}
[junit4:junit4]   2> 1598431 T3574 C617 P50923 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
[junit4:junit4]   2> 1598432 T3572 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1598433 T3574 C617 P50923 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4]   2> 1598434 T3574 C617 P50923 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4]   2> 1598435 T3574 C617 P50923 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1598436 T3497 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4]   2> 1598437 T3497 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4]   2> 1598439 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1598451 T3574 C617 P50923 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1598480 T3535 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4]   2> 1599004 T3520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1599009 T3520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"4",
[junit4:junit4]   2> 	  "numShards":"2",
[junit4:junit4]   2> 	  "shard":"shard1",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"recovering",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50923_",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50923"}
[junit4:junit4]   2> 1599053 T3571 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1599054 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1599055 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1599054 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1599054 T3555 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1599158 T3497 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4]   2> 1599183 T3497 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50927
[junit4:junit4]   2> 1599185 T3497 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4]   2> 1599187 T3497 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4]   2> 1599188 T3497 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty4-1371936080430
[junit4:junit4]   2> 1599189 T3497 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty4-1371936080430/solr.xml
[junit4:junit4]   2> 1599190 T3497 oasc.CoreContainer.<init> New CoreContainer 1776469313
[junit4:junit4]   2> 1599192 T3497 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty4-1371936080430/'
[junit4:junit4]   2> 1599194 T3497 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty4-1371936080430/'
[junit4:junit4]   2> 1599482 T3535 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4]   2> 1599485 T3535 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=4&state=recovering&nodeName=127.0.0.1:50923_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1005 
[junit4:junit4]   2> 1599505 T3497 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4]   2> 1599506 T3497 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4]   2> 1599507 T3497 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4]   2> 1599508 T3497 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4]   2> 1599510 T3497 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4]   2> 1599513 T3497 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4]   2> 1599515 T3497 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4]   2> 1599517 T3497 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4]   2> 1599520 T3497 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4]   2> 1599522 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4]   2> 1599553 T3497 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4]   2> 1599554 T3497 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50910/solr
[junit4:junit4]   2> 1599556 T3497 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4]   2> 1599562 T3497 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1599575 T3586 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a25462d name:ZooKeeperConnection Watcher:127.0.0.1:50910 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1599578 T3497 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1599590 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4]   2> 1599630 T3497 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4]   2> 1599638 T3588 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6919be61 name:ZooKeeperConnection Watcher:127.0.0.1:50910/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4]   2> 1599639 T3497 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4]   2> 1599649 T3497 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
[junit4:junit4]   2> 1600659 T3497 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50927_
[junit4:junit4]   2> 1600662 T3497 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50927_
[junit4:junit4]   2> 1600671 T3571 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4]   2> 1600672 T3571 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1600672 T3588 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4]   2> 1600691 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1600692 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4]   2> 1600704 T3519 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4]   2> 1600707 T3555 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4]   2> 1600716 T3555 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1600715 T3526 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4]   2> 1600711 T3539 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4]   2> 1600717 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1600711 T3589 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4]   2> 1600719 T3589 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2>  C617_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:50923_, base_url=http://127.0.0.1:50923}
[junit4:junit4]   2> 1601503 T3574 C617 P50923 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:50917/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4]   2> 1601504 T3574 C617 P50923 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:50923 START replicas=[http://127.0.0.1:50917/collection1/] nUpdates=100
[junit4:junit4]   2> 1601505 T3574 C617 P50923 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4]   2> 1601506 T3574 C617 P50923 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4]   2> 1601506 T3574 C617 P50923 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4]   2> 1601507 T3574 C617 P50923 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4]   2> 1601507 T3574 C617 P50923 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4]   2> 1601508 T3574 C617 P50923 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:50917/collection1/. core=collection1
[junit4:junit4]   2> 1601509 T3574 C617 P50923 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> ASYNC  NEW_CORE C618 name=collection1 org.apache.solr.core.SolrCore@40f64ab8 url=http://127.0.0.1:50917/collection1 node=127.0.0.1:50917_ C618_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50917_, base_url=http://127.0.0.1:50917, leader=true}
[junit4:junit4]   2> 1601572 T3532 C618 P50917 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4]   2> 1601578 T3531 C618 P50917 oasc.SolrCore.execute [collection1] webapp= path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=3 
[junit4:junit4]   2> 1601586 T3532 C618 P50917 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@6a946aac lockFactory=org.apache.lucene.store.NativeFSLockFactory@20f63bea),segFN=segments_1,generation=1}
[junit4:junit4]   2> 1601588 T3532 C618 P50917 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1601591 T3532 C618 P50917 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@6a946aac lockFactory=org.apache.lucene.store.NativeFSLockFactory@20f63bea),segFN=segments_1,generation=1}
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@6a946aac lockFactory=org.apache.lucene.store.NativeFSLockFactory@20f63bea),segFN=segments_2,generation=2}
[junit4:junit4]   2> 1601592 T3532 C618 P50917 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4]   2> 1601593 T3532 C618 P50917 oass.SolrIndexSearcher.<init> Opening Searcher@2ec5f050 realtime
[junit4:junit4]   2> 1601594 T3532 C618 P50917 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4]   2> 1601596 T3532 C618 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 24
[junit4:junit4]   2> 1601599 T3574 C617 P50923 oash.ReplicationHandler.inform Commits will be reserved for  10000
[junit4:junit4]   2> 1601600 T3574 C617 P50923 oash.SnapPuller.<init>  No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4]   2> 1601612 T3534 C618 P50917 oash.ReplicationHandler.inform Commits will be reserved for  10000
[junit4:junit4]   2> 1601614 T3534 C618 P50917 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=7 
[junit4:junit4]   2> 1601616 T3574 C617 P50923 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4]   2> 1601616 T3574 C617 P50923 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4]   2> 1601617 T3574 C617 P50923 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4]   2> 1601621 T3530 C618 P50917 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=1 
[junit4:junit4]   2> 1601623 T3574 C617 P50923 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4]   2> 1601626 T3574 C617 P50923 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty3/index.20130622232123591
[junit4:junit4]   2> 1601627 T3574 C617 P50923 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@78427ca8 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1312f95a) fullCopy=false
[junit4:junit4]   2> 1601634 T3535 C618 P50917 oasc.SolrCore.execute [collection1] webapp= path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=1 
[junit4:junit4]   2> 1601641 T3574 C617 P50923 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4]   2> 1601643 T3574 C617 P50923 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4]   2> 1601644 T3574 C617 P50923 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4]   2> 1601648 T3574 C617 P50923 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@227155c3 lockFactory=org.apache.lucene.store.NativeFSLockFactory@601c2328),segFN=segments_1,generation=1}
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@227155c3 lockFactory=org.apache.lucene.store.NativeFSLockFactory@601c2328),segFN=segments_2,generation=2}
[junit4:junit4]   2> 1601648 T3574 C617 P50923 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4]   2> 1601649 T3574 C617 P50923 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4]   2> 1601649 T3574 C617 P50923 oass.SolrIndexSearcher.<init> Opening Searcher@2cb0f005 main
[junit4:junit4]   2> 1601653 T3573 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2cb0f005 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4]   2> 1601655 T3574 C617 P50923 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty3/index.20130622232123591 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty3/index.20130622232123591;done=true>>]
[junit4:junit4]   2> 1601656 T3574 C617 P50923 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty3/index.20130622232123591
[junit4:junit4]   2> 1601656 T3574 C617 P50923 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty3/index.20130622232123591
[junit4:junit4]   2> 1601658 T3574 C617 P50923 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4]   2> 1601658 T3574 C617 P50923 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4]   2> 1601658 T3574 C617 P50923 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4]   2> 1601658 T3574 C617 P50923 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1601661 T3574 C617 P50923 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4]   2> 1602084 T3520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1602086 T3520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"5",
[junit4:junit4]   2> 	  "numShards":"2",
[junit4:junit4]   2> 	  "shard":null,
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"down",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50927_",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50927"}
[junit4:junit4]   2> 1602086 T3520 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4]   2> 1602086 T3520 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4]   2> 1602093 T3520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"4",
[junit4:junit4]   2> 	  "numShards":"2",
[junit4:junit4]   2> 	  "shard":"shard1",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"active",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50923_",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50923"}
[junit4:junit4]   2> 1602106 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1602106 T3571 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1602107 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1602107 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1602108 T3588 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1602108 T3555 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1602729 T3589 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty4-1371936080430/collection1
[junit4:junit4]   2> 1602730 T3589 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4]   2> 1602731 T3589 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4]   2> 1602732 T3589 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4]   2> 1602734 T3589 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty4-1371936080430/collection1/'
[junit4:junit4]   2> 1602736 T3589 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty4-1371936080430/collection1/lib/classes/' to classloader
[junit4:junit4]   2> 1602737 T3589 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty4-1371936080430/collection1/lib/README' to classloader
[junit4:junit4]   2> 1602841 T3589 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4]   2> 1602935 T3589 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4]   2> 1602938 T3589 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4]   2> 1602946 T3589 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4]   2> 1604066 T3589 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4]   2> 1604078 T3589 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4]   2> 1604086 T3589 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1604098 T3589 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4]   2> 1604204 T3589 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4]   2> 1604212 T3589 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty4-1371936080430/collection1/, dataDir=./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty4/
[junit4:junit4]   2> 1604219 T3589 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@620533a5
[junit4:junit4]   2> 1604219 T3589 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4]   2> 1604220 T3589 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty4
[junit4:junit4]   2> 1604220 T3589 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty4/index/
[junit4:junit4]   2> 1604221 T3589 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty4/index' doesn't exist. Creating new index...
[junit4:junit4]   2> 1604221 T3589 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty4/index
[junit4:junit4]   2> 1604228 T3589 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@53af6fdc lockFactory=org.apache.lucene.store.NativeFSLockFactory@45d4b35),segFN=segments_1,generation=1}
[junit4:junit4]   2> 1604229 T3589 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1604239 T3589 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4]   2> 1604240 T3589 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4]   2> 1604241 T3589 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4]   2> 1604243 T3589 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4]   2> 1604244 T3589 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4]   2> 1604245 T3589 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4]   2> 1604248 T3589 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4]   2> 1604249 T3589 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4]   2> 1604250 T3589 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4]   2> 1604266 T3589 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4]   2> 1604277 T3589 oass.SolrIndexSearcher.<init> Opening Searcher@634969c6 main
[junit4:junit4]   2> 1604278 T3589 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty4/tlog
[junit4:junit4]   2> 1604279 T3589 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4]   2> 1604280 T3589 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4]   2> 1604290 T3592 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@634969c6 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4]   2> 1604294 T3589 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4]   2> 1604295 T3589 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50927 collection:collection1 shard:shard2
[junit4:junit4]   2> 1604302 T3589 oasc.ZkController.register We are http://127.0.0.1:50927/collection1/ and leader is http://127.0.0.1:50920/collection1/
[junit4:junit4]   2> 1604303 T3589 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50927
[junit4:junit4]   2> 1604304 T3589 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4]   2> 1604304 T3589 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4]   2> ASYNC  NEW_CORE C619 name=collection1 org.apache.solr.core.SolrCore@6ed4275c url=http://127.0.0.1:50927/collection1 node=127.0.0.1:50927_ C619_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:50927_, base_url=http://127.0.0.1:50927}
[junit4:junit4]   2> 1604305 T3593 C619 P50927 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
[junit4:junit4]   2> 1604306 T3593 C619 P50927 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4]   2> 1604307 T3593 C619 P50927 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4]   2> 1604308 T3593 C619 P50927 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1604308 T3589 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1604312 T3593 C619 P50927 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1604313 T3497 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4]   2> 1604314 T3497 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4]   2> 1604314 T3497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> 1604341 T3497 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1604342 T3551 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4]   2> 1604350 T3497 oasc.AbstractFullDistribZkTestBase.waitForThingsToLevelOut Wait for recoveries to finish - wait 15 for each attempt
[junit4:junit4]   2> 1604354 T3497 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):15
[junit4:junit4]   2> 1604356 T3497 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1605232 T3520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1605235 T3520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"5",
[junit4:junit4]   2> 	  "numShards":"2",
[junit4:junit4]   2> 	  "shard":"shard2",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"recovering",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50927_",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50927"}
[junit4:junit4]   2> 1605248 T3571 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1605249 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1605250 T3555 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1605260 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1605261 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1605261 T3588 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1605346 T3551 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4]   2> 1605348 T3551 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=5&state=recovering&nodeName=127.0.0.1:50927_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1006 
[junit4:junit4]   2> 1605361 T3497 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1606366 T3497 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2>  C619_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:50927_, base_url=http://127.0.0.1:50927}
[junit4:junit4]   2> 1607350 T3593 C619 P50927 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:50920/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4]   2> 1607351 T3593 C619 P50927 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:50927 START replicas=[http://127.0.0.1:50920/collection1/] nUpdates=100
[junit4:junit4]   2> 1607352 T3593 C619 P50927 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4]   2> 1607353 T3593 C619 P50927 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4]   2> 1607353 T3593 C619 P50927 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4]   2> 1607353 T3593 C619 P50927 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4]   2> 1607354 T3593 C619 P50927 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4]   2> 1607354 T3593 C619 P50927 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:50920/collection1/. core=collection1
[junit4:junit4]   2> 1607355 T3593 C619 P50927 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4]   2> ASYNC  NEW_CORE C620 name=collection1 org.apache.solr.core.SolrCore@4a685555 url=http://127.0.0.1:50920/collection1 node=127.0.0.1:50920_ C620_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50920_, base_url=http://127.0.0.1:50920, leader=true}
[junit4:junit4]   2> 1607410 T3548 C620 P50920 oasc.SolrCore.execute [collection1] webapp= path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=3 
[junit4:junit4]   2> 1607420 T3549 C620 P50920 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4]   2> 1607429 T3497 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1607435 T3549 C620 P50920 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5675db24 lockFactory=org.apache.lucene.store.NativeFSLockFactory@6133671f),segFN=segments_1,generation=1}
[junit4:junit4]   2> 1607436 T3549 C620 P50920 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1607438 T3549 C620 P50920 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5675db24 lockFactory=org.apache.lucene.store.NativeFSLockFactory@6133671f),segFN=segments_1,generation=1}
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5675db24 lockFactory=org.apache.lucene.store.NativeFSLockFactory@6133671f),segFN=segments_2,generation=2}
[junit4:junit4]   2> 1607438 T3549 C620 P50920 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4]   2> 1607439 T3549 C620 P50920 oass.SolrIndexSearcher.<init> Opening Searcher@5171275e realtime
[junit4:junit4]   2> 1607440 T3549 C620 P50920 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4]   2> 1607441 T3549 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 21
[junit4:junit4]   2> 1607443 T3593 C619 P50927 oash.ReplicationHandler.inform Commits will be reserved for  10000
[junit4:junit4]   2> 1607444 T3593 C619 P50927 oash.SnapPuller.<init>  No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4]   2> 1607452 T3546 C620 P50920 oash.ReplicationHandler.inform Commits will be reserved for  10000
[junit4:junit4]   2> 1607453 T3546 C620 P50920 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=3 
[junit4:junit4]   2> 1607455 T3593 C619 P50927 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4]   2> 1607456 T3593 C619 P50927 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4]   2> 1607457 T3593 C619 P50927 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4]   2> 1607461 T3547 C620 P50920 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=1 
[junit4:junit4]   2> 1607464 T3593 C619 P50927 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4]   2> 1607469 T3593 C619 P50927 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty4/index.20130622232129432
[junit4:junit4]   2> 1607470 T3593 C619 P50927 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@4b071e83 lockFactory=org.apache.lucene.store.NativeFSLockFactory@9dee16c) fullCopy=false
[junit4:junit4]   2> 1607475 T3551 C620 P50920 oasc.SolrCore.execute [collection1] webapp= path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=1 
[junit4:junit4]   2> 1607479 T3593 C619 P50927 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4]   2> 1607482 T3593 C619 P50927 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4]   2> 1607483 T3593 C619 P50927 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4]   2> 1607486 T3593 C619 P50927 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@53af6fdc lockFactory=org.apache.lucene.store.NativeFSLockFactory@45d4b35),segFN=segments_1,generation=1}
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@53af6fdc lockFactory=org.apache.lucene.store.NativeFSLockFactory@45d4b35),segFN=segments_2,generation=2}
[junit4:junit4]   2> 1607488 T3593 C619 P50927 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4]   2> 1607489 T3593 C619 P50927 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4]   2> 1607490 T3593 C619 P50927 oass.SolrIndexSearcher.<init> Opening Searcher@53c025da main
[junit4:junit4]   2> 1607493 T3592 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@53c025da main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4]   2> 1607495 T3593 C619 P50927 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty4/index.20130622232129432 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty4/index.20130622232129432;done=true>>]
[junit4:junit4]   2> 1607495 T3593 C619 P50927 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty4/index.20130622232129432
[junit4:junit4]   2> 1607496 T3593 C619 P50927 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ShardSplitTest-1371936056643/jetty4/index.20130622232129432
[junit4:junit4]   2> 1607496 T3593 C619 P50927 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4]   2> 1607496 T3593 C619 P50927 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4]   2> 1607497 T3593 C619 P50927 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4]   2> 1607497 T3593 C619 P50927 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4]   2> 1607501 T3593 C619 P50927 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4]   2> 1608283 T3520 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1608285 T3520 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4]   2> 	  "operation":"state",
[junit4:junit4]   2> 	  "core_node_name":"5",
[junit4:junit4]   2> 	  "numShards":"2",
[junit4:junit4]   2> 	  "shard":"shard2",
[junit4:junit4]   2> 	  "roles":null,
[junit4:junit4]   2> 	  "state":"active",
[junit4:junit4]   2> 	  "shard_state":"active",
[junit4:junit4]   2> 	  "core":"collection1",
[junit4:junit4]   2> 	  "collection":"collection1",
[junit4:junit4]   2> 	  "shard_range":null,
[junit4:junit4]   2> 	  "node_name":"127.0.0.1:50927_",
[junit4:junit4]   2> 	  "base_url":"http://127.0.0.1:50927"}
[junit4:junit4]   2> 1608297 T3526 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1608298 T3588 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1608306 T3571 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1608308 T3539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1608308 T3519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1608309 T3555 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4]   2> 1608436 T3497 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1608439 T3497 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Recoveries finished - collection: collection1
[junit4:junit4]   2> ASYNC  NEW_CORE C621 name=collection1 org.apache.solr.core.SolrCore@2641f2dc url=http://127.0.0.1:50913/collection1 node=127.0.0.1:50913_ C621_STATE=coll:control_collection core:collection1 props:{shard=shard1, state=active, core=collection1, collection=control_collection, node_name=127.0.0.1:50913_, base_url=http://127.0.0.1:50913, leader=true}
[junit4:junit4]   2> 1608458 T3513 C621 P50913 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4]   2> 1608469 T3513 C621 P50913 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@55dd8d7b lockFactory=org.apache.lucene.store.NativeFSLockFactory@27a9f213),segFN=segments_1,generation=1}
[junit4:junit4]   2> 1608470 T3513 C621 P50913 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4]   2> 1608477 T3513 C621 P50913 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@55dd8d7b lockFactory=org.apache.lucene.store.NativeFSLockFactory@27a9f213),segFN=segments_1,generation=1}
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@55dd8d7b lockFactory=org.apache.lucene.store.NativeFSLockFactory@27a9f213),segFN=segments_2,generation=2}
[junit4:junit4]   2> 1608479 T3513 C621 P50913 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4]   2> 1608481 T3513 C621 P50913 oass.SolrIndexSearcher.<init> Opening Searcher@1943391f main
[junit4:junit4]   2> 1608483 T3513 C621 P50913 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4]   2> 1608488 T3523 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1943391f main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4]   2> 1608497 T3513 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 40
[junit4:junit4]   2>  C619_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50927_, base_url=http://127.0.0.1:50927}
[junit4:junit4]   2> 1608528 T3584 C619 P50927 oasu.SolrCmdDistributor.distribCommit Distrib commit to:[StdNode: http://127.0.0.1:50917/collection1/, StdNode: http://127.0.0.1:50923/collection1/, StdNode: http://127.0.0.1:50920/collection1/, StdNode: http://127.0.0.1:50927/collection1/] params:commit_end_point=true&commit=true&softCommit=false&waitSearcher=true&expungeDeletes=false
[junit4:junit4]   2> ASYNC  NEW_CORE C622 name=collection1 org.apache.solr.core.SolrCore@40f64ab8 url=http://127.0.0.1:50917/collection1 node=127.0.0.1:50917_ C622_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50917_, base_url=http://127.0.0.1:50917, leader=true}
[junit4:junit4]   2> 1608551 T3530 C622 P50917 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4]   2> 1608566 T3550 C620 P50920 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4]   2> ASYNC  NEW_CORE C623 name=collection1 org.apache.solr.core.SolrCore@4328c4c0 url=http://127.0.0.1:50923/collection1 node=127.0.0.1:50923_ C623_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50923_, base_url=http://127.0.0.1:50923}
[junit4:junit4]   2> 1608562 T3567 C623 P50923 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4]   2> 1608572 T3550 C620 P50920 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5675db24 lockFactory=org.apache.lucene.store.NativeFSLockFactory@6133671f),segFN=segments_2,generation=2}
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5675db24 lockFactory=org.apache.lucene.store.NativeFSLockFactory@6133671f),segFN=segments_3,generation=3}
[junit4:junit4]   2> 1608572 T3530 C622 P50917 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@6a946aac lockFactory=org.apache.lucene.store.NativeFSLockFactory@20f63bea),segFN=segments_2,generation=2}
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@6a946aac lockFactory=org.apache.lucene.store.NativeFSLockFactory@20f63bea),segFN=segments_3,generation=3}
[junit4:junit4]   2> 1608567 T3579 C619 P50927 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4]   2> 1608577 T3530 C622 P50917 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4]   2> 1608576 T3550 C620 P50920 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4]   2> 1608582 T3550 C620 P50920 oass.SolrIndexSearcher.<init> Opening Searcher@125c786a main
[junit4:junit4]   2> 1608584 T3550 C620 P50920 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4]   2> 1608575 T3567 C623 P50923 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@227155c3 lockFactory=org.apache.lucene.store.NativeFSLockFactory@601c2328),segFN=segments_2,generation=2}
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@227155c3 lockFactory=org.apache.lucene.store.NativeFSLockFactory@601c2328),segFN=segments_3,generation=3}
[junit4:junit4]   2> 1608581 T3530 C622 P50917 oass.SolrIndexSearcher.<init> Opening Searcher@1ed7caff main
[junit4:junit4]   2> 1608579 T3579 C619 P50927 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@53af6fdc lockFactory=org.apache.lucene.store.NativeFSLockFactory@45d4b35),segFN=segments_2,generation=2}
[junit4:junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@53af6fdc lockFactory=org.apache.lucene.store.NativeFSLockFactory@45d4b35),segFN=segments_3,generation=3}
[junit4:junit4]   2> 1608587 T3579 C619 P50927 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4]   2> 1608586 T3557 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@125c786a main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4]   2> 1608590 T3579 C619 P50927 oass.SolrIndexSearcher.<init> Opening Searcher@57442ef1 main
[junit4:junit4]   2> 1608589 T3567 C623 P50923 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4]   2> 1608588 T3530 C622 P50917 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4]   2> 1608588 T3541 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1ed7caff main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4]   2> 1608601 T3530 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 50
[junit4:junit4]   2> 1608599 T3567 C623 P50923 oass.SolrIndexSearcher.<init> Opening Searcher@72c35eec main
[junit4:junit4]   2> 1608596 T3592 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@57442ef1 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4]   2> 1608593 T3579 C619 P50927 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4]   2> 1608607 T3579 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 41
[junit4:junit4]   2> 1608592 T3550 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 26
[junit4:junit4]   2> 1608607 T3567 C623 P50923 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4]   2> 1608607 T3573 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@72c35eec main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4]   2> 1608613 T3567 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 51
[junit4:junit4]   2> 1608614 T3584 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 86
[junit4:junit4]   2> 1608619 T3497 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
[junit4:junit4]   2> 1608634 T3535 C622 P50917 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=1 
[junit4:junit4]   2> 1608642 T3563 C623 P50923 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0 
[junit4:junit4]   2> 1608647 T3546 C620 P50920 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=1 
[junit4:junit4]   2> 1608653 T3580 C619 P50927 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=1 
[junit4:junit4]   2> 1610667 T3513 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1438579260268740608)} 0 2
[junit4:junit4]   2> 1610688 T3580 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1438579260276080640&update.from=http://127.0.0.1:50920/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1438579260276080640)} 0 2
[junit4:junit4]   2> 1610701 T3563 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1438579260292857856&update.from=http://127.0.0.1:50917/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1438579260292857856)} 0 2
[junit4:junit4]   2> 1610703 T3535 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={update.distrib=TOLEADER&wt=javabin&version=2} {deleteByQuery=*:* (-1438579260292857856)} 0 14
[junit4:junit4]   2> 1610704 T3546 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1438579260276080640)} 0 32
[junit4:junit4]   2> 1610712 T3514 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[a!0 (1438579260314877952)]} 0 2
[junit4:junit4]   2> 1610734 T3581 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[a!0 (1438579260329558016)]} 0 1
[junit4:junit4]   2> 1610736 T3547 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50923/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[a!0 (1438579260329558016)]} 0 12
[junit4:junit4]   2> 1610737 T3564 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[a!0]} 0 20
[junit4:junit4]   2> 1610747 T3515 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[b!1 (1438579260352626688)]} 0 2
[junit4:junit4]   2> 1610759 T3565 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[b!1 (1438579260358918144)]} 0 2
[junit4:junit4]   2> 1610762 T3531 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[b!1 (1438579260358918144)]} 0 10
[junit4:junit4]   2> 1610770 T3510 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[c!2 (1438579260376743936)]} 0 2
[junit4:junit4]   2> 1610784 T3567 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[c!2 (1438579260388278272)]} 0 0
[junit4:junit4]   2> 1610786 T3532 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50923/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[c!2 (1438579260388278272)]} 0 6
[junit4:junit4]   2> 1610789 T3566 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[c!2]} 0 14
[junit4:junit4]   2> 1610794 T3511 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[d!3 (1438579260401909760)]} 0 1
[junit4:junit4]   2> 1610816 T3582 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[d!3 (1438579260419735552)]} 0 1
[junit4:junit4]   2> 1610818 T3551 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[d!3 (1438579260419735552)]} 0 8
[junit4:junit4]   2> 1610820 T3533 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[d!3]} 0 17
[junit4:junit4]   2> 1610825 T3512 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[e!4 (1438579260434415616)]} 0 1
[junit4:junit4]   2> 1610837 T3583 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[e!4 (1438579260443852800)]} 0 1
[junit4:junit4]   2> 1610839 T3548 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[e!4 (1438579260443852800)]} 0 7
[junit4:junit4]   2> 1610840 T3534 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[e!4]} 0 11
[junit4:junit4]   2> 1610845 T3513 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[f!5 (1438579260455387136)]} 0 1
[junit4:junit4]   2> 1610857 T3584 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[f!5 (1438579260464824320)]} 0 1
[junit4:junit4]   2> 1610859 T3549 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50927/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[f!5 (1438579260464824320)]} 0 6
[junit4:junit4]   2> 1610860 T3579 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[f!5]} 0 11
[junit4:junit4]   2> 1610873 T3514 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[g!6 (1438579260482650112)]} 0 7
[junit4:junit4]   2> 1610890 T3562 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[g!6 (1438579260498378752)]} 0 1
[junit4:junit4]   2> 1610893 T3530 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50927/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[g!6 (1438579260498378752)]} 0 9
[junit4:junit4]   2> 1610895 T3580 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[g!6]} 0 15
[junit4:junit4]   2> 1610900 T3515 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[h!7 (1438579260513058816)]} 0 1
[junit4:junit4]   2> 1610917 T3564 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[h!7 (1438579260525641728)]} 0 1
[junit4:junit4]   2> 1610920 T3535 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50923/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[h!7 (1438579260525641728)]} 0 9
[junit4:junit4]   2> 1610922 T3563 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[h!7]} 0 15
[junit4:junit4]   2> 1610929 T3510 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[i!8 (1438579260543467520)]} 0 1
[junit4:junit4]   2> 1610958 T3565 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[i!8 (1438579260564439040)]} 0 1
[junit4:junit4]   2> 1610960 T3531 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[i!8 (1438579260564439040)]} 0 12
[junit4:junit4]   2> 1610962 T3550 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[i!8]} 0 25
[junit4:junit4]   2> 1610967 T3512 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[j!9 (1438579260584361984)]} 0 1
[junit4:junit4]   2> 1610982 T3567 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[j!9 (1438579260594847744)]} 0 2
[junit4:junit4]   2> 1610984 T3532 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50927/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[j!9 (1438579260594847744)]} 0 8
[junit4:junit4]   2> 1610986 T3581 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[j!9]} 0 14
[junit4:junit4]   2> 1610992 T3513 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[k!10 (1438579260609527808)]} 0 2
[junit4:junit4]   2> 1611019 T3566 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[k!10 (1438579260625256448)]} 0 5
[junit4:junit4]   2> 1611021 T3533 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50927/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[k!10 (1438579260625256448)]} 0 17
[junit4:junit4]   2> 1611024 T3582 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[k!10]} 0 27
[junit4:junit4]   2> 1611029 T3514 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[l!11 (1438579260648325120)]} 0 1
[junit4:junit4]   2> 1611043 T3583 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[l!11 (1438579260658810880)]} 0 1
[junit4:junit4]   2> 1611046 T3546 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[l!11 (1438579260658810880)]} 0 8
[junit4:junit4]   2> 1611048 T3534 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[l!11]} 0 14
[junit4:junit4]   2> 1611054 T3515 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[m!12 (1438579260674539520)]} 0 1
[junit4:junit4]   2> 1611076 T3584 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[m!12 (1438579260683976704)]} 0 5
[junit4:junit4]   2> 1611078 T3547 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[m!12 (1438579260683976704)]} 0 16
[junit4:junit4]   2> 1611085 T3535 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[m!12]} 0 26
[junit4:junit4]   2> 1611091 T3510 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[n!13 (1438579260713336832)]} 0 2
[junit4:junit4]   2> 1611104 T3563 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[n!13 (1438579260722774016)]} 0 1
[junit4:junit4]   2> 1611107 T3531 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50923/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[n!13 (1438579260722774016)]} 0 8
[junit4:junit4]   2> 1611108 T3562 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[n!13]} 0 13
[junit4:junit4]   2> 1611114 T3511 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[o!14 (1438579260737454080)]} 0 1
[junit4:junit4]   2> 1611129 T3581 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[o!14 (1438579260748988416)]} 0 1
[junit4:junit4]   2> 1611131 T3548 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50927/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[o!14 (1438579260748988416)]} 0 8
[junit4:junit4]   2> 1611133 T3580 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[o!14]} 0 13
[junit4:junit4]   2> 1611139 T3512 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[p!15 (1438579260763668480)]} 0 1
[junit4:junit4]   2> 1611160 T3567 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[p!15 (1438579260782542848)]} 0 1
[junit4:junit4]   2> 1611161 T3532 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50923/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[p!15 (1438579260782542848)]} 0 5
[junit4:junit4]   2> 1611163 T3565 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[p!15]} 0 10
[junit4:junit4]   2> 1611167 T3513 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[q!16 (1438579260793028608)]} 0 1
[junit4:junit4]   2> 1611181 T3564 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[q!16 (1438579260803514368)]} 0 1
[junit4:junit4]   2> 1611183 T3533 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50923/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[q!16 (1438579260803514368)]} 0 7
[junit4:junit4]   2> 1611184 T3566 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[q!16]} 0 12
[junit4:junit4]   2> 1611190 T3514 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[r!17 (1438579260817145856)]} 0 1
[junit4:junit4]   2> 1611216 T3583 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[r!17 (1438579260830777344)]} 0 1
[junit4:junit4]   2> 1611218 T3549 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50927/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[r!17 (1438579260830777344)]} 0 18
[junit4:junit4]   2> 1611219 T3582 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[r!17]} 0 24
[junit4:junit4]   2> 1611225 T3515 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[s!18 (1438579260853846016)]} 0 1
[junit4:junit4]   2> 1611240 T3563 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[s!18 (1438579260865380352)]} 0 1
[junit4:junit4]   2> 1611242 T3534 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[s!18 (1438579260865380352)]} 0 8
[junit4:junit4]   2> 1611248 T3510 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[t!19 (1438579260879011840)]} 0 0
[junit4:junit4]   2> 1611260 T3562 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[t!19 (1438579260887400448)]} 0 1
[junit4:junit4]   2> 1611262 T3530 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[t!19 (1438579260887400448)]} 0 6
[junit4:junit4]   2> 1611264 T3550 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[t!19]} 0 11
[junit4:junit4]   2> 1611271 T3511 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[u!20 (1438579260901031936)]} 0 2
[junit4:junit4]   2> 1611289 T3579 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[u!20 (1438579260916760576)]} 0 2
[junit4:junit4]   2> 1611291 T3546 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[u!20 (1438579260916760576)]} 0 8
[junit4:junit4]   2> 1611297 T3512 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[v!21 (1438579260929343488)]} 0 1
[junit4:junit4]   2> 1611311 T3565 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[v!21 (1438579260939829248)]} 0 1
[junit4:junit4]   2> 1611312 T3535 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50923/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[v!21 (1438579260939829248)]} 0 6
[junit4:junit4]   2> 1611314 T3567 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[v!21]} 0 12
[junit4:junit4]   2> 1611320 T3513 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[w!22 (1438579260953460736)]} 0 1
[junit4:junit4]   2> 1611328 T3564 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[w!22 (1438579260958703616)]} 0 1
[junit4:junit4]   2> 1611330 T3531 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[w!22 (1438579260958703616)]} 0 6
[junit4:junit4]   2> 1611344 T3514 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[x!23 (1438579260975480832)]} 0 4
[junit4:junit4]   2> 1611352 T3584 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[x!23 (1438579260983869440)]} 0 0
[junit4:junit4]   2> 1611354 T3551 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[x!23 (1438579260983869440)]} 0 6
[junit4:junit4]   2> 1611359 T3510 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[y!24 (1438579260994355200)]} 0 1
[junit4:junit4]   2> 1611375 T3580 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[y!24 (1438579261003792384)]} 0 1
[junit4:junit4]   2> 1611377 T3547 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50927/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[y!24 (1438579261003792384)]} 0 10
[junit4:junit4]   2> 1611379 T3581 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[y!24]} 0 15
[junit4:junit4]   2> 1611384 T3511 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[z!25 (1438579261020569600)]} 0 1
[junit4:junit4]   2> 1611395 T3566 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[z!25 (1438579261028958208)]} 0 0
[junit4:junit4]   2> 1611402 T3532 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50927/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[z!25 (1438579261028958208)]} 0 11
[junit4:junit4]   2> 1611408 T3583 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[z!25]} 0 20
[junit4:junit4]   2> 1611413 T3512 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[a!26 (1438579261050978304)]} 0 2
[junit4:junit4]   2> 1611425 T3582 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[a!26 (1438579261060415488)]} 0 0
[junit4:junit4]   2> 1611427 T3548 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[a!26 (1438579261060415488)]} 0 6
[junit4:junit4]   2> 1611428 T3533 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[a!26]} 0 10
[junit4:junit4]   2> 1611433 T3513 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[b!27 (1438579261071949824)]} 0 1
[junit4:junit4]   2> 1611443 T3562 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[b!27 (1438579261079289856)]} 0 0
[junit4:junit4]   2> 1611445 T3534 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50923/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[b!27 (1438579261079289856)]} 0 6
[junit4:junit4]   2> 1611447 T3563 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[b!27]} 0 11
[junit4:junit4]   2> 1611451 T3515 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[c!28 (1438579261091872768)]} 0 1
[junit4:junit4]   2> 1611465 T3565 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[c!28 (1438579261096067072)]} 0 1
[junit4:junit4]   2> 1611467 T3530 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[c!28 (1438579261096067072)]} 0 12
[junit4:junit4]   2> 1611475 T3514 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[d!29 (1438579261117038592)]} 0 1
[junit4:junit4]   2> 1611485 T3579 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[d!29 (1438579261122281472)]} 0 1
[junit4:junit4]   2> 1611487 T3549 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[d!29 (1438579261122281472)]} 0 8
[junit4:junit4]   2> 1611491 T3510 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[e!30 (1438579261132767232)]} 0 1
[junit4:junit4]   2> 1611500 T3581 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[e!30 (1438579261140107264)]} 0 0
[junit4:junit4]   2> 1611501 T3546 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50927/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[e!30 (1438579261140107264)]} 0 4
[junit4:junit4]   2> 1611503 T3580 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[e!30]} 0 9
[junit4:junit4]   2> 1611507 T3511 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[f!31 (1438579261150593024)]} 0 1
[junit4:junit4]   2> 1611517 T3582 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[f!31 (1438579261156884480)]} 0 1
[junit4:junit4]   2> 1611521 T3551 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50927/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[f!31 (1438579261156884480)]} 0 8
[junit4:junit4]   2> 1611525 T3583 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[f!31]} 0 15
[junit4:junit4]   2> 1611530 T3512 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[g!32 (1438579261173661696)]} 0 1
[junit4:junit4]   2> 1611539 T3567 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[g!32 (1438579261179953152)]} 0 1
[junit4:junit4]   2> 1611540 T3531 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[g!32 (1438579261179953152)]} 0 6
[junit4:junit4]   2> 1611549 T3513 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[h!33 (1438579261194633216)]} 0 1
[junit4:junit4]   2> 1611557 T3566 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[h!33 (1438579261199876096)]} 0 1
[junit4:junit4]   2> 1611558 T3532 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[h!33 (1438579261199876096)]} 0 5
[junit4:junit4]   2> 1611563 T3515 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[i!34 (1438579261208264704)]} 0 1
[junit4:junit4]   2> 1611570 T3562 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[i!34 (1438579261213507584)]} 0 0
[junit4:junit4]   2> 1611572 T3533 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[i!34 (1438579261213507584)]} 0 6
[junit4:junit4]   2> 1611577 T3514 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[j!35 (1438579261223993344)]} 0 0
[junit4:junit4]   2> 1611585 T3563 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[j!35 (1438579261229236224)]} 0 1
[junit4:junit4]   2> 1611590 T3534 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[j!35 (1438579261229236224)]} 0 9
[junit4:junit4]   2> 1611600 T3510 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[k!36 (1438579261248110592)]} 0 1
[junit4:junit4]   2> 1611613 T3565 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[k!36 (1438579261253353472)]} 0 1
[junit4:junit4]   2> 1611614 T3535 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[k!36 (1438579261253353472)]} 0 9
[junit4:junit4]   2> 1611618 T3511 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[l!37 (1438579261265936384)]} 0 1
[junit4:junit4]   2> 1611628 T3584 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[l!37 (1438579261274324992)]} 0 1
[junit4:junit4]   2> 1611629 T3547 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[l!37 (1438579261274324992)]} 0 5
[junit4:junit4]   2> 1611631 T3530 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[l!37]} 0 9
[junit4:junit4]   2> 1611636 T3512 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[m!38 (1438579261284810752)]} 0 1
[junit4:junit4]   2> 1611647 T3579 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[m!38 (1438579261293199360)]} 0 0
[junit4:junit4]   2> 1611650 T3548 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50923/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[m!38 (1438579261293199360)]} 0 7
[junit4:junit4]   2> 1611655 T3564 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[m!38]} 0 15
[junit4:junit4]   2> 1611664 T3513 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[n!39 (1438579261314170880)]} 0 1
[junit4:junit4]   2> 1611675 T3567 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[n!39 (1438579261323608064)]} 0 0
[junit4:junit4]   2> 1611677 T3531 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50927/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[n!39 (1438579261323608064)]} 0 5
[junit4:junit4]   2> 1611679 T3581 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[n!39]} 0 10
[junit4:junit4]   2> 1611685 T3515 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[o!40 (1438579261337239552)]} 0 1
[junit4:junit4]   2> 1611698 T3580 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[o!40 (1438579261346676736)]} 0 0
[junit4:junit4]   2> 1611701 T3550 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[o!40 (1438579261346676736)]} 0 7
[junit4:junit4]   2> 1611703 T3532 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[o!40]} 0 13
[junit4:junit4]   2> 1611710 T3514 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[p!41 (1438579261362405376)]} 0 1
[junit4:junit4]   2> 1611730 T3566 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[p!41 (1438579261379182592)]} 0 1
[junit4:junit4]   2> 1611731 T3533 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[p!41 (1438579261379182592)]} 0 8
[junit4:junit4]   2> 1611732 T3549 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[p!41]} 0 17
[junit4:junit4]   2> 1611738 T3510 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[q!42 (1438579261391765504)]} 0 1
[junit4:junit4]   2> 1611752 T3562 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[q!42 (1438579261402251264)]} 0 1
[junit4:junit4]   2> 1611753 T3534 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[q!42 (1438579261402251264)]} 0 6
[junit4:junit4]   2> 1611757 T3512 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[r!43 (1438579261411688448)]} 0 1
[junit4:junit4]   2> 1611763 T3582 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[r!43 (1438579261415882752)]} 0 0
[junit4:junit4]   2> 1611765 T3546 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[r!43 (1438579261415882752)]} 0 5
[junit4:junit4]   2> 1611769 T3513 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[s!44 (1438579261425319936)]} 0 1
[junit4:junit4]   2> 1611779 T3563 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[s!44 (1438579261431611392)]} 0 1
[junit4:junit4]   2> 1611781 T3535 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50927/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[s!44 (1438579261431611392)]} 0 6
[junit4:junit4]   2> 1611784 T3583 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[s!44]} 0 11
[junit4:junit4]   2> 1611795 T3515 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[t!45 (1438579261451534336)]} 0 1
[junit4:junit4]   2> 1611806 T3565 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[t!45 (1438579261458874368)]} 0 1
[junit4:junit4]   2> 1611807 T3530 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[t!45 (1438579261458874368)]} 0 6
[junit4:junit4]   2> 1611812 T3514 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[u!46 (1438579261470408704)]} 0 1
[junit4:junit4]   2> 1611827 T3584 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[u!46 (1438579261480894464)]} 0 1
[junit4:junit4]   2> 1611830 T3551 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50923/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[u!46 (1438579261480894464)]} 0 9
[junit4:junit4]   2> 1611832 T3564 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[u!46]} 0 14
[junit4:junit4]   2> 1611839 T3510 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[v!47 (1438579261497671680)]} 0 1
[junit4:junit4]   2> 1611858 T3567 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[v!47 (1438579261508157440)]} 0 5
[junit4:junit4]   2> 1611860 T3531 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50927/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[v!47 (1438579261508157440)]} 0 12
[junit4:junit4]   2> 1611862 T3579 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[v!47]} 0 17
[junit4:junit4]   2> 1611870 T3511 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[w!48 (1438579261529128960)]} 0 2
[junit4:junit4]   2> 1611891 T3562 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[w!48 (1438579261546954752)]} 0 2
[junit4:junit4]   2> 1611894 T3533 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50923/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[w!48 (1438579261546954752)]} 0 10
[junit4:junit4]   2> 1611896 T3566 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[w!48]} 0 16
[junit4:junit4]   2> 1611902 T3512 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[x!49 (1438579261564780544)]} 0 1
[junit4:junit4]   2> 1611915 T3581 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[x!49 (1438579261574217728)]} 0 1
[junit4:junit4]   2> 1611917 T3547 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[x!49 (1438579261574217728)]} 0 7
[junit4:junit4]   2> 1611919 T3534 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[x!49]} 0 12
[junit4:junit4]   2> 1611925 T3513 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[y!50 (1438579261587849216)]} 0 2
[junit4:junit4]   2> 1611946 T3582 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[y!50 (1438579261600432128)]} 0 1
[junit4:junit4]   2> 1611948 T3550 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[y!50 (1438579261600432128)]} 0 13
[junit4:junit4]   2> 1611953 T3515 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[z!51 (1438579261617209344)]} 0 1
[junit4:junit4]   2> 1611965 T3564 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[z!51 (1438579261626646528)]} 0 1
[junit4:junit4]   2> 1611967 T3535 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50923/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[z!51 (1438579261626646528)]} 0 7
[junit4:junit4]   2> 1611970 T3563 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[z!51]} 0 13
[junit4:junit4]   2> 1611976 T3514 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[a!52 (1438579261641326592)]} 0 1
[junit4:junit4]   2> 1611987 T3583 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[a!52 (1438579261649715200)]} 0 1
[junit4:junit4]   2> 1611989 T3549 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[a!52 (1438579261649715200)]} 0 7
[junit4:junit4]   2> 1611995 T3530 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[a!52]} 0 15
[junit4:junit4]   2> 1612005 T3510 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[b!53 (1438579261672783872)]} 0 1
[junit4:junit4]   2> 1612019 T3562 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[b!53 (1438579261683269632)]} 0 1
[junit4:junit4]   2> 1612021 T3531 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50923/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[b!53 (1438579261683269632)]} 0 7
[junit4:junit4]   2> 1612022 T3567 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[b!53]} 0 11
[junit4:junit4]   2> 1612027 T3511 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[c!54 (1438579261695852544)]} 0 0
[junit4:junit4]   2> 1612040 T3566 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[c!54 (1438579261704241152)]} 0 1
[junit4:junit4]   2> 1612042 T3532 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50927/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[c!54 (1438579261704241152)]} 0 7
[junit4:junit4]   2> 1612043 T3584 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[c!54]} 0 11
[junit4:junit4]   2> 1612048 T3512 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[d!55 (1438579261716824064)]} 0 1
[junit4:junit4]   2> 1612061 T3579 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[d!55 (1438579261722066944)]} 0 2
[junit4:junit4]   2> 1612064 T3546 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[d!55 (1438579261722066944)]} 0 12
[junit4:junit4]   2> 1612070 T3513 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[e!56 (1438579261739892736)]} 0 1
[junit4:junit4]   2> 1612082 T3580 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[e!56 (1438579261749329920)]} 0 1
[junit4:junit4]   2> 1612083 T3551 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50923/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[e!56 (1438579261749329920)]} 0 5
[junit4:junit4]   2> 1612084 T3565 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[e!56]} 0 10
[junit4:junit4]   2> 1612089 T3515 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[f!57 (1438579261760864256)]} 0 1
[junit4:junit4]   2> 1612098 T3581 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50920/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[f!57 (1438579261766107136)]} 0 1
[junit4:junit4]   2> 1612100 T3548 C620 P50920 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[f!57 (1438579261766107136)]} 0 7
[junit4:junit4]   2> 1612104 T3514 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[g!58 (1438579261776592896)]} 0 1
[junit4:junit4]   2> 1612114 T3564 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[g!58 (1438579261783932928)]} 0 1
[junit4:junit4]   2> 1612116 T3533 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50927/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[g!58 (1438579261783932928)]} 0 5
[junit4:junit4]   2> 1612117 T3582 C619 P50927 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[g!58]} 0 9
[junit4:junit4]   2> 1612126 T3510 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[h!59 (1438579261798612992)]} 0 2
[junit4:junit4]   2> 1612135 T3563 C623 P50923 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:50917/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[h!59 (1438579261804904448)]} 0 1
[junit4:junit4]   2> 1612137 T3534 C622 P50917 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[h!59 (1438579261804904448)]} 0 6
[junit4:junit4]   2> 1612141 T3511 C621 P50913 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[i!60 (1438579261815

[...truncated too long message...]

/leader_elect/shard1/election/89911225723518988-5-n_0000000003 (0)
[junit4:junit4]   1>        /solr/collections/collection1/leader_elect/shard1/election/89911225723518986-4-n_0000000002 (0)
[junit4:junit4]   1>     /solr/collections/collection1/leaders (1)
[junit4:junit4]   1>      /solr/collections/collection1/leaders/shard1 (0)
[junit4:junit4]   1>      DATA:
[junit4:junit4]   1>          {
[junit4:junit4]   1>            "core":"collection1",
[junit4:junit4]   1>            "node_name":"127.0.0.1:51176_",
[junit4:junit4]   1>            "base_url":"http://127.0.0.1:51176"}
[junit4:junit4]   1>    /solr/collections/control_collection (3)
[junit4:junit4]   1>    DATA:
[junit4:junit4]   1>        {"configName":"conf1"}
[junit4:junit4]   1>     /solr/collections/control_collection/shards (0)
[junit4:junit4]   1>     /solr/collections/control_collection/leader_elect (1)
[junit4:junit4]   1>      /solr/collections/control_collection/leader_elect/shard1 (1)
[junit4:junit4]   1>       /solr/collections/control_collection/leader_elect/shard1/election (1)
[junit4:junit4]   1>        /solr/collections/control_collection/leader_elect/shard1/election/89911225723518979-1-n_0000000000 (0)
[junit4:junit4]   1>     /solr/collections/control_collection/leaders (1)
[junit4:junit4]   1>      /solr/collections/control_collection/leaders/shard1 (0)
[junit4:junit4]   1>      DATA:
[junit4:junit4]   1>          {
[junit4:junit4]   1>            "core":"collection1",
[junit4:junit4]   1>            "node_name":"127.0.0.1:51165_",
[junit4:junit4]   1>            "base_url":"http://127.0.0.1:51165"}
[junit4:junit4]   1>   /solr/overseer_elect (2)
[junit4:junit4]   1>    /solr/overseer_elect/election (4)
[junit4:junit4]   1>     /solr/overseer_elect/election/89911225723518986-127.0.0.1:51176_-n_0000000003 (0)
[junit4:junit4]   1>     /solr/overseer_elect/election/89911225723518990-127.0.0.1:51169_-n_0000000005 (0)
[junit4:junit4]   1>     /solr/overseer_elect/election/89911225723518988-127.0.0.1:51183_-n_0000000004 (0)
[junit4:junit4]   1>     /solr/overseer_elect/election/89911225723518979-127.0.0.1:51165_-n_0000000000 (0)
[junit4:junit4]   1>    /solr/overseer_elect/leader (0)
[junit4:junit4]   1>    DATA:
[junit4:junit4]   1>        {"id":"89911225723518979-127.0.0.1:51165_-n_0000000000"}
[junit4:junit4]   1>  /zookeeper (1)
[junit4:junit4]   1>  DATA:
[junit4:junit4]   1>      
[junit4:junit4]   1> 
[junit4:junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SyncSliceTest -Dtests.method=testDistribSearch -Dtests.seed=ED5D37BBB19952BE -Dtests.slow=true -Dtests.locale=fi_FI -Dtests.timezone=America/Argentina/San_Luis -Dtests.file.encoding=ISO-8859-1
[junit4:junit4] FAILURE  128s | SyncSliceTest.testDistribSearch <<<
[junit4:junit4]    > Throwable #1: java.lang.AssertionError: shard1 is not consistent.  Got 305 from http://127.0.0.1:51169/collection1lastClient and got 259 from http://127.0.0.1:51176/collection1
[junit4:junit4]    > 	at __randomizedtesting.SeedInfo.seed([ED5D37BBB19952BE:6CBBB9A3C6C63282]:0)
[junit4:junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1018)
[junit4:junit4]    > 	at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:238)
[junit4:junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
[junit4:junit4]    > 	at java.lang.Thread.run(Thread.java:724)
[junit4:junit4]   2> 2073776 T4089 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4:junit4]   2> 128110 T4088 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 2 leaked thread(s).
[junit4:junit4]   2> 2074558 T4183 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4]   2> 2074558 T4183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4]   2> 2074559 T4183 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4]   2> NOTE: test params are: codec=Lucene42: {range_facet_l=Pulsing41(freqCutoff=10 minBlockSize=79 maxBlockSize=254), text=PostingsFormat(name=Memory doPackFST= false), _version_=PostingsFormat(name=SimpleText), rnd_b=PostingsFormat(name=MockRandom), intDefault=Pulsing41(freqCutoff=10 minBlockSize=79 maxBlockSize=254), timestamp=Pulsing41(freqCutoff=10 minBlockSize=79 maxBlockSize=254), id=PostingsFormat(name=MockRandom), a_t=Pulsing41(freqCutoff=10 minBlockSize=79 maxBlockSize=254), range_facet_sl=PostingsFormat(name=MockRandom), range_facet_si=Pulsing41(freqCutoff=10 minBlockSize=79 maxBlockSize=254), other_tl1=Pulsing41(freqCutoff=10 minBlockSize=79 maxBlockSize=254), multiDefault=Pulsing41(freqCutoff=10 minBlockSize=79 maxBlockSize=254), a_si=PostingsFormat(name=SimpleText)}, docValues:{timestamp=DocValuesFormat(name=Lucene42)}, sim=DefaultSimilarity, locale=fi_FI, timezone=America/Argentina/San_Luis
[junit4:junit4]   2> NOTE: Mac OS X 10.8.4 x86_64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=2,threads=3,free=173366768,total=536870912
[junit4:junit4]   2> NOTE: All tests run in this JVM: [TestReload, TestXIncludeConfig, XmlUpdateRequestHandlerTest, TestPropInject, HighlighterConfigTest, ZkControllerTest, ZkCLITest, TestSolrIndexConfig, SuggesterTest, SimpleFacetsTest, NumericFieldsTest, TestAtomicUpdateErrorCases, TestSchemaVersionResource, DirectUpdateHandlerTest, OpenCloseCoreStressTest, RequiredFieldsTest, TimeZoneUtilsTest, TestGroupingSearch, JsonLoaderTest, TestPerFieldSimilarity, TestMergePolicyConfig, PeerSyncTest, SolrCmdDistributorTest, ClusterStateUpdateTest, SolrPluginUtilsTest, SolrCoreCheckLockOnStartupTest, SolrInfoMBeanTest, DisMaxRequestHandlerTest, TestExtendedDismaxParser, ResponseLogComponentTest, TestPostingsSolrHighlighter, OverseerTest, ChaosMonkeySafeLeaderTest, TestFoldingMultitermQuery, OutputWriterTest, TestFastWriter, SolrCoreTest, SearchHandlerTest, TestCollationField, FileBasedSpellCheckerTest, TestSurroundQueryParser, TestManagedSchemaFieldResource, TestDFRSimilarityFactory, PathHierarchyTokenizerFactoryTest, MultiTermTest, TestSolrDeletionPolicy1, TestOmitPositions, TestCoreDiscovery, TestSolrXml, SolrIndexConfigTest, SimplePostToolTest, WordBreakSolrSpellCheckerTest, TestBinaryField, FieldMutatingUpdateProcessorTest, TestRemoteStreaming, TestUtils, TestFastLRUCache, TestManagedSchema, SoftAutoCommitTest, TestDefaultSearchFieldResource, TestCodecSupport, TestPropInjectDefaults, TestSuggestSpellingConverter, TestFieldCollectionResource, TestSchemaResource, DistanceFunctionTest, MBeansHandlerTest, SchemaVersionSpecificBehaviorTest, StandardRequestHandlerTest, CSVRequestHandlerTest, TestFieldResource, DOMUtilTest, TestDynamicFieldCollectionResource, TestTrie, BasicDistributedZkTest, SignatureUpdateProcessorFactoryTest, CircularListTest, TestPartialUpdateDeduplication, TestDistributedSearch, TestCopyFieldCollectionResource, CollectionsAPIDistributedZkTest, TestUniqueKeyFieldResource, TestClassNameShortening, IndexSchemaTest, TestRangeQuery, EchoParamsTest, NotRequiredUniqueKeyTest, TestPHPSerializedResponseWriter, PolyFieldTest, TestSolrQueryParser, DebugComponentTest, TestPseudoReturnFields, TestSolrDeletionPolicy2, BadCopyFieldTest, StatelessScriptUpdateProcessorFactoryTest, FastVectorHighlighterTest, BadComponentTest, TestReversedWildcardFilterFactory, DocValuesMultiTest, SpellingQueryConverterTest, BinaryUpdateRequestHandlerTest, DirectUpdateHandlerOptimizeTest, TestRealTimeGet, TestArbitraryIndexDir, RequestHandlersTest, UUIDFieldTest, TestCloudManagedSchemaAddField, AlternateDirectoryTest, TestSolr4Spatial, TestSchemaSimilarityResource, TermVectorComponentTest, TestFuzzyAnalyzedSuggestions, TestSolrQueryParserDefaultOperatorResource, TestNumberUtils, TestSolrJ, LeaderElectionTest, PingRequestHandlerTest, ResourceLoaderTest, TestFunctionQuery, PrimUtilsTest, SliceStateUpdateTest, SpellCheckComponentTest, PrimitiveFieldTypeTest, PreAnalyzedUpdateProcessorTest, AutoCommitTest, TestValueSourceCache, TestQuerySenderListener, FullSolrCloudDistribCmdsTest, TestQueryUtils, CacheHeaderTest, ShardSplitTest, TestFieldTypeResource, RAMDirectoryFactoryTest, FieldAnalysisRequestHandlerTest, UpdateRequestProcessorFactoryTest, TestRTGBase, TestJoin, TestDocSet, DocumentBuilderTest, TestSchemaNameResource, TestWordDelimiterFilterFactory, TestLuceneMatchVersion, BasicDistributedZk2Test, UnloadDistributedZkTest, SyncSliceTest]
[junit4:junit4] Completed in 129.17s, 1 test, 1 failure <<< FAILURES!

[...truncated 456 lines...]
BUILD FAILED
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:386: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:366: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:39: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:181: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:437: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1243: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:887: There were test failures: 297 suites, 1253 tests, 1 error, 1 failure, 18 ignored (6 assumptions)

Total time: 107 minutes 16 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0 -XX:+UseCompressedOops -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure