You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/06/14 11:18:51 UTC
[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.7.0) - Build # 560 -
Failure!
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/560/
Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseG1GC
2 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
Error Message:
ERROR: SolrIndexSearcher opens=43 closes=42
Stack Trace:
java.lang.AssertionError: ERROR: SolrIndexSearcher opens=43 closes=42
at __randomizedtesting.SeedInfo.seed([F7FC16EE94CE27DB]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:254)
at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:102)
at sun.reflect.GeneratedMethodAccessor23.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:601)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:700)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:722)
REGRESSION: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch
Error Message:
collection already exists: awholynewcollection_0
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: collection already exists: awholynewcollection_0
at __randomizedtesting.SeedInfo.seed([F7FC16EE94CE27DB:761A98F6E39147E7]:0)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:424)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:264)
at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:306)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1400)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:438)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:146)
at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:601)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:722)
Build Log:
[...truncated 9608 lines...]
[junit4:junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
[junit4:junit4] 2> 1942660 T3198 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
[junit4:junit4] 2> 1942705 T3198 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4:junit4] 2> Creating dataDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./solrtest-ChaosMonkeySafeLeaderTest-1371199565867
[junit4:junit4] 2> 1942711 T3198 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4:junit4] 2> 1942718 T3199 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4:junit4] 2> 1942819 T3198 oasc.ZkTestServer.run start zk server on port:49979
[junit4:junit4] 2> 1942831 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1942841 T3205 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c92321b name:ZooKeeperConnection Watcher:127.0.0.1:49979 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1942842 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1942843 T3198 oascc.SolrZkClient.makePath makePath: /solr
[junit4:junit4] 2> 1942866 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1942883 T3207 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15110732 name:ZooKeeperConnection Watcher:127.0.0.1:49979/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1942884 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1942885 T3198 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4:junit4] 2> 1942903 T3198 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4:junit4] 2> 1942919 T3198 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4:junit4] 2> 1942929 T3198 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4:junit4] 2> 1942944 T3198 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 1942946 T3198 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 1942972 T3198 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4:junit4] 2> 1942974 T3198 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4:junit4] 2> 1942990 T3198 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4:junit4] 2> 1942992 T3198 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4:junit4] 2> 1943002 T3198 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4:junit4] 2> 1943004 T3198 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4:junit4] 2> 1943018 T3198 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4:junit4] 2> 1943021 T3198 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4:junit4] 2> 1943039 T3198 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 1943040 T3198 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 1943077 T3198 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 1943079 T3198 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 1943096 T3198 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 1943097 T3198 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 1943111 T3198 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4:junit4] 2> 1943113 T3198 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4:junit4] 2> 1943958 T3198 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1943970 T3198 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49982
[junit4:junit4] 2> 1943972 T3198 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1943973 T3198 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1943974 T3198 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371199566286
[junit4:junit4] 2> 1943976 T3198 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371199566286/solr.xml
[junit4:junit4] 2> 1943977 T3198 oasc.CoreContainer.<init> New CoreContainer 1098154721
[junit4:junit4] 2> 1943981 T3198 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371199566286/'
[junit4:junit4] 2> 1943983 T3198 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371199566286/'
[junit4:junit4] 2> 1944444 T3198 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1944446 T3198 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1944446 T3198 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1944447 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1944448 T3198 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1944448 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1944449 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1944450 T3198 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1944451 T3198 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1944452 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1944507 T3198 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1944509 T3198 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49979/solr
[junit4:junit4] 2> 1944510 T3198 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1944513 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1944619 T3218 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4cc9cb99 name:ZooKeeperConnection Watcher:127.0.0.1:49979 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1944622 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1944642 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1944682 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1944694 T3220 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@36853c9b name:ZooKeeperConnection Watcher:127.0.0.1:49979/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1944695 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1944703 T3198 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4:junit4] 2> 1944742 T3198 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4:junit4] 2> 1944760 T3198 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4:junit4] 2> 1944769 T3198 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49982_
[junit4:junit4] 2> 1944774 T3198 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49982_
[junit4:junit4] 2> 1944788 T3198 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4:junit4] 2> 1944809 T3198 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4:junit4] 2> 1944821 T3198 oasc.Overseer.start Overseer (id=89862934749904899-127.0.0.1:49982_-n_0000000000) starting
[junit4:junit4] 2> 1944845 T3198 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4:junit4] 2> 1944867 T3222 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4:junit4] 2> 1944878 T3198 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4:junit4] 2> 1944887 T3198 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4:junit4] 2> 1944897 T3198 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1944937 T3221 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4:junit4] 2> 1944940 T3223 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1944940 T3223 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1946463 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1946467 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":null,
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49982_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49982"}
[junit4:junit4] 2> 1946468 T3221 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
[junit4:junit4] 2> 1946469 T3221 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1946485 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1946950 T3223 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371199566286/collection1
[junit4:junit4] 2> 1946951 T3223 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4:junit4] 2> 1946954 T3223 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1946960 T3223 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4:junit4] 2> 1946967 T3223 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371199566286/collection1/'
[junit4:junit4] 2> 1946970 T3223 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371199566286/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1946972 T3223 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371199566286/collection1/lib/README' to classloader
[junit4:junit4] 2> 1947135 T3223 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1947311 T3223 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1947317 T3223 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1947330 T3223 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1949267 T3223 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1949300 T3223 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1949307 T3223 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1949381 T3223 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1949395 T3223 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1949415 T3223 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1949449 T3223 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1949450 T3223 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1949451 T3223 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1949466 T3223 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1949467 T3223 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1949468 T3223 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1949597 T3223 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371199566286/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/control/data/
[junit4:junit4] 2> 1949598 T3223 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@453521ec
[junit4:junit4] 2> 1949599 T3223 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1949602 T3223 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/control/data
[junit4:junit4] 2> 1949603 T3223 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/control/data/index/
[junit4:junit4] 2> 1949605 T3223 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/control/data/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1949608 T3223 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/control/data/index
[junit4:junit4] 2> 1949620 T3223 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/control/data/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1949622 T3223 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1949641 T3223 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1949643 T3223 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1949644 T3223 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1949646 T3223 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1949648 T3223 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1949648 T3223 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1949649 T3223 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1949650 T3223 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1949652 T3223 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1949687 T3223 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1949727 T3223 oass.SolrIndexSearcher.<init> Opening Searcher@19ec2bb2 main
[junit4:junit4] 2> 1949730 T3223 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1949731 T3223 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1949741 T3224 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@19ec2bb2 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1949743 T3223 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1949744 T3223 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49982 collection:control_collection shard:shard1
[junit4:junit4] 2> 1949747 T3223 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4:junit4] 2> 1949797 T3223 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 1949808 T3223 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 1949810 T3223 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 1949810 T3223 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49982/collection1/
[junit4:junit4] 2> 1949811 T3223 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 1949812 T3223 oasc.SyncStrategy.syncToMe http://127.0.0.1:49982/collection1/ has no replicas
[junit4:junit4] 2> 1949812 T3223 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49982/collection1/
[junit4:junit4] 2> 1949814 T3223 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4:junit4] 2> 1951028 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1951053 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1951083 T3223 oasc.ZkController.register We are http://127.0.0.1:49982/collection1/ and leader is http://127.0.0.1:49982/collection1/
[junit4:junit4] 2> 1951084 T3223 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49982
[junit4:junit4] 2> 1951084 T3223 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 1951085 T3223 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1951085 T3223 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1951100 T3223 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1951108 T3198 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1951109 T3198 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1951110 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1951150 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:
[junit4:junit4] 2> 1951155 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1951165 T3227 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@716ed6e0 name:ZooKeeperConnection Watcher:127.0.0.1:49979/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1951166 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1951174 T3198 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1951186 T3198 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
[junit4:junit4] 2> 1951840 T3198 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1951850 T3198 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49986
[junit4:junit4] 2> 1951853 T3198 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1951853 T3198 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1951854 T3198 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371199574347
[junit4:junit4] 2> 1951857 T3198 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371199574347/solr.xml
[junit4:junit4] 2> 1951860 T3198 oasc.CoreContainer.<init> New CoreContainer 1214856542
[junit4:junit4] 2> 1951862 T3198 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371199574347/'
[junit4:junit4] 2> 1951863 T3198 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371199574347/'
[junit4:junit4] 2> 1952083 T3198 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1952084 T3198 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1952085 T3198 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1952085 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1952086 T3198 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1952087 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1952087 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1952088 T3198 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1952089 T3198 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1952090 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1952119 T3198 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1952121 T3198 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49979/solr
[junit4:junit4] 2> 1952122 T3198 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1952125 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1952140 T3238 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@448ff644 name:ZooKeeperConnection Watcher:127.0.0.1:49979 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1952141 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1952148 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1952175 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1952182 T3240 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25d262f9 name:ZooKeeperConnection Watcher:127.0.0.1:49979/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1952182 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1952204 T3198 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1952580 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1952582 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:49982__collection1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49982_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49982"}
[junit4:junit4] 2> 1952601 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1952606 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1952609 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1953223 T3198 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49986_
[junit4:junit4] 2> 1953228 T3198 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49986_
[junit4:junit4] 2> 1953236 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1953237 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1953237 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1953245 T3220 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 1953247 T3240 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 1953247 T3227 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 1953264 T3241 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1953264 T3241 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1954139 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1954142 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":null,
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49986_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49986"}
[junit4:junit4] 2> 1954142 T3221 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 3
[junit4:junit4] 2> 1954143 T3221 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1954165 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1954174 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1954179 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1954269 T3241 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371199574347/collection1
[junit4:junit4] 2> 1954270 T3241 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1954273 T3241 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1954273 T3241 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1954281 T3241 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371199574347/collection1/'
[junit4:junit4] 2> 1954291 T3241 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371199574347/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1954293 T3241 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371199574347/collection1/lib/README' to classloader
[junit4:junit4] 2> 1954411 T3241 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1954571 T3241 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1954577 T3241 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1954586 T3241 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1956251 T3241 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1956277 T3241 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1956286 T3241 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1956346 T3241 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1956384 T3241 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1956400 T3241 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1956422 T3241 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1956424 T3241 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1956425 T3241 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1956429 T3241 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1956429 T3241 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1956430 T3241 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1956431 T3241 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371199574347/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty1/
[junit4:junit4] 2> 1956431 T3241 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@453521ec
[junit4:junit4] 2> 1956432 T3241 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1956434 T3241 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty1
[junit4:junit4] 2> 1956435 T3241 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty1/index/
[junit4:junit4] 2> 1956436 T3241 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty1/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1956437 T3241 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty1/index
[junit4:junit4] 2> 1956447 T3241 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty1/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1956448 T3241 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1956455 T3241 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1956456 T3241 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1956458 T3241 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1956460 T3241 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1956462 T3241 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1956463 T3241 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1956464 T3241 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1956466 T3241 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1956468 T3241 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1956497 T3241 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1956525 T3241 oass.SolrIndexSearcher.<init> Opening Searcher@1e58581a main
[junit4:junit4] 2> 1956531 T3241 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1956535 T3241 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1956547 T3242 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1e58581a main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1956551 T3241 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1956552 T3241 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49986 collection:collection1 shard:shard1
[junit4:junit4] 2> 1956554 T3241 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4:junit4] 2> 1956582 T3241 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 1956595 T3241 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 1956596 T3241 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 1956597 T3241 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49986/collection1/
[junit4:junit4] 2> 1956598 T3241 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 1956599 T3241 oasc.SyncStrategy.syncToMe http://127.0.0.1:49986/collection1/ has no replicas
[junit4:junit4] 2> 1956600 T3241 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49986/collection1/
[junit4:junit4] 2> 1956600 T3241 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4:junit4] 2> 1957204 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1957235 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1957236 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1957237 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1957292 T3241 oasc.ZkController.register We are http://127.0.0.1:49986/collection1/ and leader is http://127.0.0.1:49986/collection1/
[junit4:junit4] 2> 1957293 T3241 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49986
[junit4:junit4] 2> 1957294 T3241 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 1957294 T3241 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1957295 T3241 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1957299 T3241 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1957303 T3198 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1957304 T3198 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1957305 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1958012 T3198 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1958019 T3198 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49989
[junit4:junit4] 2> 1958027 T3198 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1958029 T3198 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1958030 T3198 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371199580500
[junit4:junit4] 2> 1958032 T3198 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371199580500/solr.xml
[junit4:junit4] 2> 1958033 T3198 oasc.CoreContainer.<init> New CoreContainer 1463290390
[junit4:junit4] 2> 1958035 T3198 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371199580500/'
[junit4:junit4] 2> 1958036 T3198 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371199580500/'
[junit4:junit4] 2> 1958291 T3198 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1958294 T3198 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1958295 T3198 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1958296 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1958296 T3198 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1958297 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1958298 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1958298 T3198 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1958299 T3198 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1958300 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1958329 T3198 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1958331 T3198 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49979/solr
[junit4:junit4] 2> 1958331 T3198 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1958335 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1958339 T3254 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ec8f70b name:ZooKeeperConnection Watcher:127.0.0.1:49979 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1958341 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1958347 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1958373 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1958384 T3256 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e74f6e0 name:ZooKeeperConnection Watcher:127.0.0.1:49979/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1958385 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1958407 T3198 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1958766 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1958769 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:49986__collection1",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49986_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49986"}
[junit4:junit4] 2> 1958786 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1958792 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1958804 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1958804 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1959423 T3198 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49989_
[junit4:junit4] 2> 1959428 T3198 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49989_
[junit4:junit4] 2> 1959436 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1959437 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1959436 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1959437 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1959447 T3220 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1959448 T3240 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1959449 T3227 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1959452 T3256 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1959467 T3257 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1959468 T3257 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1960309 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1960311 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":null,
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49989_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49989"}
[junit4:junit4] 2> 1960311 T3221 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4] 2> 1960312 T3221 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 1960325 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1960328 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1960330 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1960331 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1960472 T3257 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371199580500/collection1
[junit4:junit4] 2> 1960473 T3257 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1960477 T3257 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1960477 T3257 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1960482 T3257 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371199580500/collection1/'
[junit4:junit4] 2> 1960486 T3257 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371199580500/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1960488 T3257 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371199580500/collection1/lib/README' to classloader
[junit4:junit4] 2> 1960632 T3257 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1960774 T3257 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1960778 T3257 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1960787 T3257 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1962480 T3257 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1962507 T3257 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1962518 T3257 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1962587 T3257 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1962601 T3257 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1962611 T3257 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1962616 T3257 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1962620 T3257 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1962621 T3257 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1962625 T3257 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1962626 T3257 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1962626 T3257 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1962628 T3257 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371199580500/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty2/
[junit4:junit4] 2> 1962630 T3257 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@453521ec
[junit4:junit4] 2> 1962633 T3257 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1962637 T3257 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty2
[junit4:junit4] 2> 1962639 T3257 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty2/index/
[junit4:junit4] 2> 1962640 T3257 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty2/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1962641 T3257 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty2/index
[junit4:junit4] 2> 1962654 T3257 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty2/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1962656 T3257 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1962664 T3257 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1962666 T3257 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1962667 T3257 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1962670 T3257 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1962672 T3257 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1962673 T3257 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1962674 T3257 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1962676 T3257 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1962677 T3257 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1962717 T3257 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1962740 T3257 oass.SolrIndexSearcher.<init> Opening Searcher@c5f1989 main
[junit4:junit4] 2> 1962743 T3257 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1962743 T3257 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1962756 T3258 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@c5f1989 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1962763 T3257 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1962764 T3257 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49989 collection:collection1 shard:shard2
[junit4:junit4] 2> 1962767 T3257 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4:junit4] 2> 1962795 T3257 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 1962824 T3257 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 1962824 T3257 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 1962825 T3257 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49989/collection1/
[junit4:junit4] 2> 1962825 T3257 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 1962826 T3257 oasc.SyncStrategy.syncToMe http://127.0.0.1:49989/collection1/ has no replicas
[junit4:junit4] 2> 1962826 T3257 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49989/collection1/
[junit4:junit4] 2> 1962827 T3257 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4:junit4] 2> 1963358 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1963398 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1963399 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1963400 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1963400 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1963436 T3257 oasc.ZkController.register We are http://127.0.0.1:49989/collection1/ and leader is http://127.0.0.1:49989/collection1/
[junit4:junit4] 2> 1963437 T3257 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49989
[junit4:junit4] 2> 1963438 T3257 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 1963438 T3257 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1963440 T3257 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1963450 T3257 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1963459 T3198 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1963460 T3198 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1963462 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1964288 T3198 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1964296 T3198 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49992
[junit4:junit4] 2> 1964301 T3198 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1964303 T3198 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1964304 T3198 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371199586665
[junit4:junit4] 2> 1964305 T3198 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371199586665/solr.xml
[junit4:junit4] 2> 1964306 T3198 oasc.CoreContainer.<init> New CoreContainer 2075581472
[junit4:junit4] 2> 1964308 T3198 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371199586665/'
[junit4:junit4] 2> 1964309 T3198 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371199586665/'
[junit4:junit4] 2> 1964638 T3198 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1964641 T3198 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1964642 T3198 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1964643 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1964644 T3198 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1964645 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1964646 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1964647 T3198 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1964648 T3198 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1964649 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1964683 T3198 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1964685 T3198 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49979/solr
[junit4:junit4] 2> 1964686 T3198 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1964689 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1964696 T3270 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6925b437 name:ZooKeeperConnection Watcher:127.0.0.1:49979 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1964698 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1964705 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1964729 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1964736 T3272 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b4abf9f name:ZooKeeperConnection Watcher:127.0.0.1:49979/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1964737 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1964759 T3198 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1964927 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1964931 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:49989__collection1",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49989_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49989"}
[junit4:junit4] 2> 1964945 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1964945 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1964946 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1964946 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1964946 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1965773 T3198 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49992_
[junit4:junit4] 2> 1965778 T3198 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49992_
[junit4:junit4] 2> 1965788 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1965794 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1965795 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1965796 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1965802 T3227 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1965811 T3256 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1965821 T3272 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1965822 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1965831 T3220 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1965837 T3240 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1965858 T3273 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1965859 T3273 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1966472 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1966475 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":null,
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49992_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49992"}
[junit4:junit4] 2> 1966475 T3221 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4] 2> 1966476 T3221 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard3
[junit4:junit4] 2> 1966509 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1966510 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1966511 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1966510 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1966510 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1966865 T3273 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371199586665/collection1
[junit4:junit4] 2> 1966865 T3273 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1966867 T3273 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1966867 T3273 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1966872 T3273 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371199586665/collection1/'
[junit4:junit4] 2> 1966874 T3273 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371199586665/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1966875 T3273 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371199586665/collection1/lib/README' to classloader
[junit4:junit4] 2> 1966993 T3273 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1967182 T3273 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1967187 T3273 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1967200 T3273 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1969005 T3273 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1969031 T3273 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1969040 T3273 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1969097 T3273 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1969107 T3273 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1969117 T3273 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1969121 T3273 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1969123 T3273 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1969124 T3273 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1969127 T3273 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1969127 T3273 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1969128 T3273 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1969128 T3273 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371199586665/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty3/
[junit4:junit4] 2> 1969129 T3273 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@453521ec
[junit4:junit4] 2> 1969130 T3273 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1969131 T3273 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty3
[junit4:junit4] 2> 1969132 T3273 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty3/index/
[junit4:junit4] 2> 1969133 T3273 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty3/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1969134 T3273 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty3/index
[junit4:junit4] 2> 1969143 T3273 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty3/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1969143 T3273 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1969156 T3273 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1969159 T3273 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1969161 T3273 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1969163 T3273 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1969164 T3273 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1969165 T3273 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1969166 T3273 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1969167 T3273 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1969169 T3273 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1969192 T3273 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1969205 T3273 oass.SolrIndexSearcher.<init> Opening Searcher@60361865 main
[junit4:junit4] 2> 1969207 T3273 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1969208 T3273 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1969218 T3273 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1969220 T3273 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49992 collection:collection1 shard:shard3
[junit4:junit4] 2> 1969222 T3274 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@60361865 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1969224 T3273 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
[junit4:junit4] 2> 1969242 T3273 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 1969251 T3273 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 1969253 T3273 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 1969255 T3273 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49992/collection1/
[junit4:junit4] 2> 1969257 T3273 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 1969258 T3273 oasc.SyncStrategy.syncToMe http://127.0.0.1:49992/collection1/ has no replicas
[junit4:junit4] 2> 1969260 T3273 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49992/collection1/
[junit4:junit4] 2> 1969261 T3273 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
[junit4:junit4] 2> 1969561 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1969595 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1969596 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1969597 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1969598 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1969614 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1969641 T3273 oasc.ZkController.register We are http://127.0.0.1:49992/collection1/ and leader is http://127.0.0.1:49992/collection1/
[junit4:junit4] 2> 1969642 T3273 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49992
[junit4:junit4] 2> 1969643 T3273 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 1969643 T3273 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1969644 T3273 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1969649 T3273 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1969654 T3198 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1969656 T3198 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1969659 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1970543 T3198 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1970558 T3198 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49995
[junit4:junit4] 2> 1970561 T3198 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1970562 T3198 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1970563 T3198 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371199592866
[junit4:junit4] 2> 1970564 T3198 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371199592866/solr.xml
[junit4:junit4] 2> 1970566 T3198 oasc.CoreContainer.<init> New CoreContainer 401623176
[junit4:junit4] 2> 1970567 T3198 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371199592866/'
[junit4:junit4] 2> 1970569 T3198 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371199592866/'
[junit4:junit4] 2> 1970814 T3198 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1970815 T3198 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1970816 T3198 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1970817 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1970817 T3198 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1970818 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1970819 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1970819 T3198 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1970820 T3198 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1970825 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1970869 T3198 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1970871 T3198 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49979/solr
[junit4:junit4] 2> 1970872 T3198 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1970876 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1970883 T3286 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a193d24 name:ZooKeeperConnection Watcher:127.0.0.1:49979 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1970884 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1970891 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1970931 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1970937 T3288 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5d1f2655 name:ZooKeeperConnection Watcher:127.0.0.1:49979/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1970938 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1970955 T3198 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1971146 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1971148 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:49992__collection1",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard3",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49992_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49992"}
[junit4:junit4] 2> 1971169 T3288 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1971171 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1971172 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1971173 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1971173 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1971174 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1971972 T3198 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49995_
[junit4:junit4] 2> 1971976 T3198 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49995_
[junit4:junit4] 2> 1971989 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1971990 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1971991 T3288 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1971993 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1971994 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1972002 T3272 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1972004 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1972023 T3288 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1972025 T3227 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1972026 T3240 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1972027 T3220 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1972030 T3256 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1972056 T3289 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1972057 T3289 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1972688 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1972694 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":null,
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49995_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49995"}
[junit4:junit4] 2> 1972695 T3221 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4] 2> 1972695 T3221 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1972709 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1972710 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1972711 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1972709 T3288 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1972709 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1972711 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1973062 T3289 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371199592866/collection1
[junit4:junit4] 2> 1973063 T3289 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1973065 T3289 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1973066 T3289 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1973071 T3289 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371199592866/collection1/'
[junit4:junit4] 2> 1973075 T3289 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371199592866/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1973076 T3289 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371199592866/collection1/lib/README' to classloader
[junit4:junit4] 2> 1973218 T3289 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1973385 T3289 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1973390 T3289 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1973405 T3289 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1975169 T3289 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1975203 T3289 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1975213 T3289 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1975271 T3289 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1975284 T3289 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1975297 T3289 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1975301 T3289 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1975304 T3289 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1975306 T3289 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1975313 T3289 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1975319 T3289 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1975322 T3289 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1975323 T3289 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371199592866/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty4/
[junit4:junit4] 2> 1975323 T3289 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@453521ec
[junit4:junit4] 2> 1975324 T3289 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1975326 T3289 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty4
[junit4:junit4] 2> 1975330 T3289 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty4/index/
[junit4:junit4] 2> 1975332 T3289 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty4/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1975334 T3289 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty4/index
[junit4:junit4] 2> 1975346 T3289 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty4/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1975347 T3289 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1975355 T3289 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1975357 T3289 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1975359 T3289 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1975361 T3289 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1975363 T3289 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1975363 T3289 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1975364 T3289 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1975366 T3289 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1975367 T3289 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1975404 T3289 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1975418 T3289 oass.SolrIndexSearcher.<init> Opening Searcher@428bcbe8 main
[junit4:junit4] 2> 1975420 T3289 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1975421 T3289 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1975431 T3290 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@428bcbe8 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1975436 T3289 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1975437 T3289 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49995 collection:collection1 shard:shard1
[junit4:junit4] 2> 1975448 T3289 oasc.ZkController.register We are http://127.0.0.1:49995/collection1/ and leader is http://127.0.0.1:49986/collection1/
[junit4:junit4] 2> 1975449 T3289 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49995
[junit4:junit4] 2> 1975450 T3289 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 1975451 T3289 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C1071 name=collection1 org.apache.solr.core.SolrCore@46c798ea url=http://127.0.0.1:49995/collection1 node=127.0.0.1:49995_ C1071_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:49995_, base_url=http://127.0.0.1:49995}
[junit4:junit4] 2> 1975452 T3291 C1071 P49995 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 1975455 T3291 C1071 P49995 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 1975455 T3291 C1071 P49995 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 1975456 T3291 C1071 P49995 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1975457 T3289 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1975459 T3291 C1071 P49995 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1975464 T3198 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1975466 T3198 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1975467 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1975533 T3234 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 127.0.0.1:49995__collection1, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 1975749 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1975753 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:49995__collection1",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49995_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49995"}
[junit4:junit4] 2> 1975778 T3288 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1975779 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1975780 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1975781 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1975779 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1975781 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1976359 T3198 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1976369 T3198 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49999
[junit4:junit4] 2> 1976375 T3198 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1976377 T3198 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1976378 T3198 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371199598666
[junit4:junit4] 2> 1976380 T3198 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371199598666/solr.xml
[junit4:junit4] 2> 1976391 T3198 oasc.CoreContainer.<init> New CoreContainer 552172687
[junit4:junit4] 2> 1976392 T3198 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371199598666/'
[junit4:junit4] 2> 1976393 T3198 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371199598666/'
[junit4:junit4] 2> 1976536 T3234 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 127.0.0.1:49995__collection1, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 1976537 T3234 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=127.0.0.1:49995__collection1&state=recovering&nodeName=127.0.0.1:49995_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1004
[junit4:junit4] 2> 1976687 T3198 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1976689 T3198 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1976690 T3198 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1976692 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1976693 T3198 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1976694 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1976695 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1976696 T3198 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1976697 T3198 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1976699 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1976743 T3198 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1976745 T3198 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49979/solr
[junit4:junit4] 2> 1976746 T3198 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1976755 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1976759 T3303 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a911d4d name:ZooKeeperConnection Watcher:127.0.0.1:49979 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1976760 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1976767 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1976807 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1976815 T3305 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b3cb06d name:ZooKeeperConnection Watcher:127.0.0.1:49979/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1976816 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1976835 T3198 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1977859 T3198 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49999_
[junit4:junit4] 2> 1977864 T3198 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49999_
[junit4:junit4] 2> 1977875 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1977876 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1977889 T3288 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1977890 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1977890 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1977896 T3305 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1977900 T3272 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1977901 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1977903 T3220 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1977907 T3227 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1977907 T3256 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1977911 T3288 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1977912 T3240 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1977929 T3306 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1977930 T3306 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> C1071_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:49995_, base_url=http://127.0.0.1:49995}
[junit4:junit4] 2> 1978541 T3291 C1071 P49995 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:49986/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 1978542 T3291 C1071 P49995 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:49995 START replicas=[http://127.0.0.1:49986/collection1/] nUpdates=100
[junit4:junit4] 2> 1978543 T3291 C1071 P49995 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 1978546 T3291 C1071 P49995 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 1978548 T3291 C1071 P49995 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 1978548 T3291 C1071 P49995 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 1978548 T3291 C1071 P49995 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 1978549 T3291 C1071 P49995 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:49986/collection1/. core=collection1
[junit4:junit4] 2> 1978549 T3291 C1071 P49995 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C1072 name=collection1 org.apache.solr.core.SolrCore@b0686f2 url=http://127.0.0.1:49986/collection1 node=127.0.0.1:49986_ C1072_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:49986_, base_url=http://127.0.0.1:49986, leader=true}
[junit4:junit4] 2> 1978597 T3232 C1072 P49986 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 1978604 T3232 C1072 P49986 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty1/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1978607 T3232 C1072 P49986 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1978611 T3234 C1072 P49986 oasc.SolrCore.execute [collection1] webapp= path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0
[junit4:junit4] 2> 1978613 T3232 C1072 P49986 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty1/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty1/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 1978615 T3232 C1072 P49986 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1978617 T3232 C1072 P49986 oass.SolrIndexSearcher.<init> Opening Searcher@42554dbe realtime
[junit4:junit4] 2> 1978618 T3232 C1072 P49986 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1978619 T3232 C1072 P49986 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 22
[junit4:junit4] 2> 1978622 T3291 C1071 P49995 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1978622 T3291 C1071 P49995 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 1978635 T3235 C1072 P49986 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1978637 T3235 C1072 P49986 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=6
[junit4:junit4] 2> 1978638 T3291 C1071 P49995 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 1978639 T3291 C1071 P49995 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 1978639 T3291 C1071 P49995 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 1978644 T3236 C1072 P49986 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=1
[junit4:junit4] 2> 1978647 T3291 C1071 P49995 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 1978651 T3291 C1071 P49995 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty4/index.20130614044641807
[junit4:junit4] 2> 1978660 T3291 C1071 P49995 oash.SnapPuller.fetchLatestIndex Starting download to org.apache.lucene.store.NIOFSDirectory@/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty4/index.20130614044641807 lockFactory=org.apache.lucene.store.NativeFSLockFactory@6742bfe0 fullCopy=false
[junit4:junit4] 2> 1978668 T3231 C1072 P49986 oasc.SolrCore.execute [collection1] webapp= path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0
[junit4:junit4] 2> 1978671 T3291 C1071 P49995 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 1978675 T3291 C1071 P49995 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 1978676 T3291 C1071 P49995 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 1978682 T3291 C1071 P49995 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty4/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty4/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 1978683 T3291 C1071 P49995 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1978684 T3291 C1071 P49995 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 1978686 T3291 C1071 P49995 oass.SolrIndexSearcher.<init> Opening Searcher@17f22147 main
[junit4:junit4] 2> 1978688 T3290 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@17f22147 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 1978690 T3291 C1071 P49995 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty4/index.20130614044641807 [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty4/index.20130614044641807;done=true>>]
[junit4:junit4] 2> 1978691 T3291 C1071 P49995 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty4/index.20130614044641807
[junit4:junit4] 2> 1978691 T3291 C1071 P49995 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty4/index.20130614044641807
[junit4:junit4] 2> 1978694 T3291 C1071 P49995 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 1978694 T3291 C1071 P49995 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 1978695 T3291 C1071 P49995 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1978695 T3291 C1071 P49995 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1978699 T3291 C1071 P49995 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 1978822 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1978825 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":null,
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49999_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49999"}
[junit4:junit4] 2> 1978826 T3221 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4] 2> 1978826 T3221 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 1978837 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:49995__collection1",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49995_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49995"}
[junit4:junit4] 2> 1978851 T3288 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1978852 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1978853 T3305 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1978854 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1978855 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1978855 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1978856 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1978935 T3306 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371199598666/collection1
[junit4:junit4] 2> 1978936 T3306 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1978939 T3306 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1978940 T3306 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1978946 T3306 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371199598666/collection1/'
[junit4:junit4] 2> 1978949 T3306 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371199598666/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1978950 T3306 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371199598666/collection1/lib/README' to classloader
[junit4:junit4] 2> 1979131 T3306 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1979326 T3306 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1979338 T3306 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1979348 T3306 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1980635 T3306 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1980653 T3306 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1980660 T3306 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1980709 T3306 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1980720 T3306 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1980730 T3306 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1980735 T3306 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1980736 T3306 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1980737 T3306 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1980741 T3306 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1980741 T3306 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1980742 T3306 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1980743 T3306 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371199598666/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty5/
[junit4:junit4] 2> 1980744 T3306 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@453521ec
[junit4:junit4] 2> 1980745 T3306 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1980747 T3306 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty5
[junit4:junit4] 2> 1980749 T3306 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty5/index/
[junit4:junit4] 2> 1980751 T3306 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty5/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1980753 T3306 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty5/index
[junit4:junit4] 2> 1980764 T3306 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty5/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1980765 T3306 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1980774 T3306 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1980775 T3306 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1980777 T3306 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1980780 T3306 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1980782 T3306 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1980782 T3306 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1980796 T3306 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1980802 T3306 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1980804 T3306 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1980835 T3306 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1980851 T3306 oass.SolrIndexSearcher.<init> Opening Searcher@49ab9f3d main
[junit4:junit4] 2> 1980853 T3306 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1980854 T3306 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1980860 T3309 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@49ab9f3d main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1980865 T3306 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1980865 T3306 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49999 collection:collection1 shard:shard2
[junit4:junit4] 2> 1980876 T3306 oasc.ZkController.register We are http://127.0.0.1:49999/collection1/ and leader is http://127.0.0.1:49989/collection1/
[junit4:junit4] 2> 1980882 T3306 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49999
[junit4:junit4] 2> 1980882 T3306 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 1980883 T3306 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C1073 name=collection1 org.apache.solr.core.SolrCore@52c0ed6 url=http://127.0.0.1:49999/collection1 node=127.0.0.1:49999_ C1073_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:49999_, base_url=http://127.0.0.1:49999}
[junit4:junit4] 2> 1980884 T3310 C1073 P49999 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 1980885 T3310 C1073 P49999 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 1980885 T3310 C1073 P49999 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 1980885 T3310 C1073 P49999 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1980899 T3310 C1073 P49999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1980900 T3306 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1980910 T3198 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1980912 T3198 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1980913 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1980940 T3248 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 127.0.0.1:49999__collection1, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 1981561 T3198 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1981572 T3198 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50006
[junit4:junit4] 2> 1981574 T3198 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1981574 T3198 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1981575 T3198 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371199604107
[junit4:junit4] 2> 1981576 T3198 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371199604107/solr.xml
[junit4:junit4] 2> 1981577 T3198 oasc.CoreContainer.<init> New CoreContainer 1126637518
[junit4:junit4] 2> 1981578 T3198 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371199604107/'
[junit4:junit4] 2> 1981578 T3198 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371199604107/'
[junit4:junit4] 2> 1981801 T3198 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1981803 T3198 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1981804 T3198 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1981805 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1981806 T3198 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1981807 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1981808 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1981810 T3198 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1981811 T3198 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1981813 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1981853 T3198 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1981856 T3198 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49979/solr
[junit4:junit4] 2> 1981857 T3198 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1981861 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1981868 T3322 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9d3b8dd name:ZooKeeperConnection Watcher:127.0.0.1:49979 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1981869 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1981876 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1981903 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1981905 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:49999__collection1",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49999_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49999"}
[junit4:junit4] 2> 1981921 T3288 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1981922 T3305 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1981922 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1981923 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1981923 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1981923 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1981924 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1981942 T3248 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 127.0.0.1:49999__collection1, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 1981944 T3248 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=127.0.0.1:49999__collection1&state=recovering&nodeName=127.0.0.1:49999_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1004
[junit4:junit4] 2> 1981956 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1981963 T3324 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3da13b66 name:ZooKeeperConnection Watcher:127.0.0.1:49979/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1981964 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1981978 T3198 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1982993 T3198 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50006_
[junit4:junit4] 2> 1983005 T3198 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50006_
[junit4:junit4] 2> 1983016 T3288 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1983018 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1983018 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1983021 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1983023 T3324 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1983025 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1983026 T3272 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1983027 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1983042 T3288 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1983046 T3240 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1983047 T3220 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1983048 T3256 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1983054 T3227 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1983060 T3305 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1983061 T3305 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1983091 T3325 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1983092 T3325 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1983451 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1983454 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":null,
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50006_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50006"}
[junit4:junit4] 2> 1983455 T3221 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4] 2> 1983456 T3221 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard3
[junit4:junit4] 2> 1983478 T3288 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1983479 T3324 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1983480 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1983480 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1983481 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1983480 T3305 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1983481 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1983480 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> C1073_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:49999_, base_url=http://127.0.0.1:49999}
[junit4:junit4] 2> 1983957 T3310 C1073 P49999 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:49989/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 1983958 T3310 C1073 P49999 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:49999 START replicas=[http://127.0.0.1:49989/collection1/] nUpdates=100
[junit4:junit4] 2> 1983959 T3310 C1073 P49999 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 1983959 T3310 C1073 P49999 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 1983959 T3310 C1073 P49999 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 1983960 T3310 C1073 P49999 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 1983960 T3310 C1073 P49999 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 1983960 T3310 C1073 P49999 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:49989/collection1/. core=collection1
[junit4:junit4] 2> 1983961 T3310 C1073 P49999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C1074 name=collection1 org.apache.solr.core.SolrCore@3ea54b43 url=http://127.0.0.1:49989/collection1 node=127.0.0.1:49989_ C1074_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:49989_, base_url=http://127.0.0.1:49989, leader=true}
[junit4:junit4] 2> 1984012 T3251 C1074 P49989 oasc.SolrCore.execute [collection1] webapp= path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=1
[junit4:junit4] 2> 1984013 T3252 C1074 P49989 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 1984020 T3252 C1074 P49989 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty2/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1984022 T3252 C1074 P49989 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1984026 T3252 C1074 P49989 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty2/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty2/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 1984027 T3252 C1074 P49989 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1984030 T3252 C1074 P49989 oass.SolrIndexSearcher.<init> Opening Searcher@742d2da0 realtime
[junit4:junit4] 2> 1984031 T3252 C1074 P49989 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1984031 T3252 C1074 P49989 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 18
[junit4:junit4] 2> 1984033 T3310 C1073 P49999 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1984034 T3310 C1073 P49999 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 1984041 T3247 C1074 P49989 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1984043 T3247 C1074 P49989 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=4
[junit4:junit4] 2> 1984045 T3310 C1073 P49999 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 1984046 T3310 C1073 P49999 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 1984046 T3310 C1073 P49999 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 1984052 T3250 C1074 P49989 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=2
[junit4:junit4] 2> 1984054 T3310 C1073 P49999 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 1984061 T3310 C1073 P49999 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty5/index.20130614044647214
[junit4:junit4] 2> 1984065 T3310 C1073 P49999 oash.SnapPuller.fetchLatestIndex Starting download to org.apache.lucene.store.NIOFSDirectory@/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty5/index.20130614044647214 lockFactory=org.apache.lucene.store.NativeFSLockFactory@10fe6432 fullCopy=false
[junit4:junit4] 2> 1984073 T3248 C1074 P49989 oasc.SolrCore.execute [collection1] webapp= path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=1
[junit4:junit4] 2> 1984076 T3310 C1073 P49999 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 1984080 T3310 C1073 P49999 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 1984080 T3310 C1073 P49999 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 1984089 T3310 C1073 P49999 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty5/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty5/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 1984091 T3310 C1073 P49999 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1984092 T3310 C1073 P49999 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 1984093 T3310 C1073 P49999 oass.SolrIndexSearcher.<init> Opening Searcher@796e5e5d main
[junit4:junit4] 2> 1984095 T3309 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@796e5e5d main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 1984096 T3310 C1073 P49999 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty5/index.20130614044647214 [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty5/index.20130614044647214;done=true>>]
[junit4:junit4] 2> 1984097 T3310 C1073 P49999 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty5/index.20130614044647214
[junit4:junit4] 2> 1984097 T3310 C1073 P49999 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty5/index.20130614044647214
[junit4:junit4] 2> 1984098 T3325 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371199604107/collection1
[junit4:junit4] 2> 1984098 T3325 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1984101 T3310 C1073 P49999 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 1984101 T3310 C1073 P49999 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 1984102 T3310 C1073 P49999 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1984102 T3310 C1073 P49999 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1984103 T3325 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1984104 T3325 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1984106 T3310 C1073 P49999 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 1984110 T3325 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371199604107/collection1/'
[junit4:junit4] 2> 1984112 T3325 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371199604107/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1984113 T3325 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371199604107/collection1/lib/README' to classloader
[junit4:junit4] 2> 1984252 T3325 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1984405 T3325 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1984409 T3325 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1984418 T3325 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1985021 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1985025 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:49999__collection1",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49999_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49999"}
[junit4:junit4] 2> 1985048 T3288 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1985049 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1985050 T3324 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1985051 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1985051 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1985052 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1985054 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1985052 T3305 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1986018 T3325 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1986035 T3325 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1986041 T3325 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1986087 T3325 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1986097 T3325 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1986106 T3325 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1986111 T3325 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1986113 T3325 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1986114 T3325 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1986117 T3325 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1986117 T3325 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1986118 T3325 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1986119 T3325 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371199604107/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty6/
[junit4:junit4] 2> 1986120 T3325 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@453521ec
[junit4:junit4] 2> 1986121 T3325 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1986122 T3325 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty6
[junit4:junit4] 2> 1986123 T3325 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty6/index/
[junit4:junit4] 2> 1986124 T3325 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty6/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1986125 T3325 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty6/index
[junit4:junit4] 2> 1986135 T3325 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371199565865/jetty6/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1986140 T3325 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1986150 T3325 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1986151 T3325 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1986152 T3325 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1986153 T3325 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1986155 T3325 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1986156 T3325 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1986162 T3325 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1986163 T3325 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1986165 T3325 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1986186 T3325 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1986205 T3325 oass.SolrIndexSearcher.<init> Opening Searcher@5dedb5d1 main
[junit4:junit4] 2> 1986207 T3325 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1986208 T3325 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1986217 T3328 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5dedb5d1 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1986221 T3325 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1986223 T3325 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50006 collection:collection1 shard:shard3
[junit4:junit4] 2> 1986234 T3325 oasc.ZkController.register We are http://127.0.0.1:50006/collection1/ and leader is http://127.0.0.1:49992/collection1/
[junit4:junit4] 2> 1986234 T3325 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50006
[junit4:junit4] 2> 1986235 T3325 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 1986235 T3325 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C1075 name=collection1 org.apache.solr.core.SolrCore@2390e203 url=http://127.0.0.1:50006/collection1 node=127.0.0.1:50006_ C1075_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:50006_, base_url=http://127.0.0.1:50006}
[junit4:junit4] 2> 1986237 T3329 C1075 P50006 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 1986239 T3329 C1075 P50006 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 1986239 T3329 C1075 P50006 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 1986240 T3329 C1075 P50006 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1986242 T3325 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1986243 T3329 C1075 P50006 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1986265 T3198 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1986266 T3198 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1986266 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1986290 T3263 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 127.0.0.1:50006__collection1, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 1986771 T3221 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1986780 T3221 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:50006__collection1",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard3",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50006_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50006"}
[junit4:junit4] 2> 1986806 T3288 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1986807 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1986808 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1986808 T3305 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1986808 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1986808 T3324 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1986808 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1986809 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1987051 T3198 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1987070 T3198 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50013
[junit4:junit4] 2> 1987074 T3198 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1987075 T3198 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1987075 T3198 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371199609451
[junit4:junit4] 2> 1987076 T3198 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371199609451/solr.xml
[junit4:junit4] 2> 1987078 T3198 oasc.CoreContainer.<init> New CoreContainer 1447116838
[junit4:junit4] 2> 1987079 T3198 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371199609451/'
[junit4:junit4] 2> 1987080 T3198 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371199609451/'
[junit4:junit4] 2> 1987293 T3263 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 127.0.0.1:50006__collection1, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 1987294 T3263 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=127.0.0.1:50006__collection1&state=recovering&nodeName=127.0.0.1:50006_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1004
[junit4:junit4] 2> 1987380 T3198 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1987381 T3198 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1987382 T3198 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1987382 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1987383 T3198 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1987384 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1987384 T3198 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1987385 T3198 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1987386 T3198 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1987386 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1987415 T3198 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1987417 T3198 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49979/solr
[junit4:junit4] 2> 1987418 T3198 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1987421 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1987428 T3341 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3c60c770 name:ZooKeeperConnection Watcher:127.0.0.1:49979 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1987429 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1987435 T3198 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1987474 T3198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1987480 T3343 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f1e8b8c name:ZooKeeperConnection Watcher:127.0.0.1:49979/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1987480 T3198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1987495 T3198 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1988514 T3198 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50013_
[junit4:junit4] 2> 1988525 T3198 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50013_
[junit4:junit4] 2> 1988541 T3288 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1988542 T3324 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1988544 T3227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1988545 T3240 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1988546 T3220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1988545 T3256 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1988553 T3305 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1988554 T3305 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4] 2> 1988560 T3272 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1988560 T3272 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4] 2> 1988563 T3343 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1988568 T3288 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1988581 T3240 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1988583 T3220 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1988583 T3227 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1988586 T3324 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1988589 T3256 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1988617 T3344 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1988618 T3344 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> C1075_STATE=coll:collection1 core:collection1 props:{shard=shard3, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:50006_, base_url=http://127.0.0.1:50006}
[junit4:junit4] 2> 1989299 T3329 C1075 P50006 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:49992/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 1989299 T3329 C1075 P50006 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:50006 START replicas=[http://127.0.0.1:49992/collection1/] nUpdates=100
[junit4:junit4] 2> 1989300 T3329 C1075 P50006 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 1989301 T3329 C1075 P50006 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 1989302 T3329 C1075 P50006 oasc.RecoveryStrategy.doRecovery Starting Rep
[...truncated too long message...]
utedZkTest-jetty4-1371200081130/awholynewcollection_0_shard2_replica1/data/index [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1371200081130/awholynewcollection_0_shard2_replica1/data/index;done=false>>]
[junit4:junit4] 2> 2513186 T3969 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1371200081130/awholynewcollection_0_shard2_replica1/data/index
[junit4:junit4] 2> 2513187 T3969 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1371200081130/awholynewcollection_0_shard2_replica1/data [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1371200081130/awholynewcollection_0_shard2_replica1/data;done=false>>]
[junit4:junit4] 2> 2513188 T3969 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1371200081130/awholynewcollection_0_shard2_replica1/data
[junit4:junit4] 2> 2513189 T4198 oasc.Overseer$ClusterStateUpdater.amILeader According to ZK I (id=89862967886807052-127.0.0.1:50348_-n_0000000004) am no longer a leader.
[junit4:junit4] 2> 2513197 T3971 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x13f41e4a75b000c, likely client has closed socket
[junit4:junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
[junit4:junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4:junit4] 2> at java.lang.Thread.run(Thread.java:722)
[junit4:junit4] 2>
[junit4:junit4] 2> 2513212 T3969 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
[junit4:junit4] 2> 2513276 T3969 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
[junit4:junit4] 2> 2513282 T3969 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:50332 50332
[junit4:junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4:junit4] 2> NOTE: reproduce with: ant test -Dtestcase=CollectionsAPIDistributedZkTest -Dtests.method=testDistribSearch -Dtests.seed=F7FC16EE94CE27DB -Dtests.slow=true -Dtests.locale=es_HN -Dtests.timezone=America/Argentina/Cordoba -Dtests.file.encoding=US-ASCII
[junit4:junit4] ERROR 65.2s | CollectionsAPIDistributedZkTest.testDistribSearch <<<
[junit4:junit4] > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: collection already exists: awholynewcollection_0
[junit4:junit4] > at __randomizedtesting.SeedInfo.seed([F7FC16EE94CE27DB:761A98F6E39147E7]:0)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:424)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:264)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:306)
[junit4:junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1400)
[junit4:junit4] > at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:438)
[junit4:junit4] > at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:146)
[junit4:junit4] > at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
[junit4:junit4] > at java.lang.Thread.run(Thread.java:722)
[junit4:junit4] 2> 2513517 T3969 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4:junit4] 2> 65198 T3968 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 2 leaked thread(s).
[junit4:junit4] 2> 2514497 T4051 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 2514498 T4051 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 2514498 T4051 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> NOTE: test params are: codec=Lucene42: {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=es_HN, timezone=America/Argentina/Cordoba
[junit4:junit4] 2> NOTE: Mac OS X 10.8.4 x86_64/Oracle Corporation 1.7.0_21 (64-bit)/cpus=2,threads=5,free=164494128,total=536870912
[junit4:junit4] 2> NOTE: All tests run in this JVM: [TestLMJelinekMercerSimilarityFactory, TestSolrCoreProperties, SliceStateTest, TestRTGBase, StatsComponentTest, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, NotRequiredUniqueKeyTest, TestReload, SystemInfoHandlerTest, TestDynamicFieldCollectionResource, TestExtendedDismaxParser, TestDocumentBuilder, TestOmitPositions, OverseerCollectionProcessorTest, IndexReaderFactoryTest, ClusterStateUpdateTest, TestFieldTypeResource, TestSurroundQueryParser, TestCloudManagedSchemaAddField, TestMultiCoreConfBootstrap, SchemaVersionSpecificBehaviorTest, ZkNodePropsTest, DOMUtilTest, TestJoin, SignatureUpdateProcessorFactoryTest, CoreContainerCoreInitFailuresTest, TestDistributedGrouping, TimeZoneUtilsTest, UUIDFieldTest, TestFieldTypeCollectionResource, OutputWriterTest, CopyFieldTest, TestStressVersions, TestUpdate, IndexSchemaTest, TestWordDelimiterFilterFactory, BadIndexSchemaTest, TestTrie, SortByFunctionTest, OpenCloseCoreStressTest, TestZkChroot, HighlighterTest, TestSolrDeletionPolicy1, RAMDirectoryFactoryTest, TestSolrXml, TestQueryTypes, TestUtils, PingRequestHandlerTest, TestSolrQueryParser, TestDefaultSearchFieldResource, TestFieldCollectionResource, TestSolr4Spatial, BasicFunctionalityTest, SpellCheckComponentTest, TestSolrIndexConfig, TestCharFilters, DirectSolrSpellCheckerTest, SuggesterWFSTTest, TestBinaryField, ClusterStateTest, ShardRoutingTest, PrimUtilsTest, TestFaceting, SimpleFacetsTest, MBeansHandlerTest, TestShardHandlerFactory, ScriptEngineTest, SuggesterTSTTest, JSONWriterTest, TestBM25SimilarityFactory, TestFastOutputStream, DirectUpdateHandlerTest, TestLazyCores, ShowFileRequestHandlerTest, FieldMutatingUpdateProcessorTest, TestFiltering, LeaderElectionIntegrationTest, TestFoldingMultitermQuery, TestSolrDeletionPolicy2, TestQueryUtils, HardAutoCommitTest, JsonLoaderTest, TestHashPartitioner, TestBadConfig, TestStressRecovery, TestSolrJ, HighlighterConfigTest, SolrCoreTest, TestPropInject, EchoParamsTest, LegacyHTMLStripCharFilterTest, TestSchemaResource, UniqFieldsUpdateProcessorFactoryTest, TestMaxScoreQueryParser, TestFastLRUCache, DistributedQueryElevationComponentTest, CachingDirectoryFactoryTest, SuggesterTest, SpellingQueryConverterTest, CoreAdminHandlerTest, TestRecovery, ReturnFieldsTest, SOLR749Test, TestSchemaNameResource, TestQuerySenderNoQuery, ConvertedLegacyTest, FileBasedSpellCheckerTest, CacheHeaderTest, PathHierarchyTokenizerFactoryTest, CircularListTest, DistributedSpellCheckComponentTest, SpatialFilterTest, DocumentAnalysisRequestHandlerTest, DocValuesMultiTest, SolrRequestParserTest, BadCopyFieldTest, TestDynamicFieldResource, TestAtomicUpdateErrorCases, PrimitiveFieldTypeTest, TestNumberUtils, TestCopyFieldCollectionResource, AliasIntegrationTest, ZkSolrClientTest, SolrPluginUtilsTest, TestRangeQuery, DirectUpdateHandlerOptimizeTest, AnalysisAfterCoreReloadTest, AutoCommitTest, PreAnalyzedUpdateProcessorTest, IndexSchemaRuntimeFieldTest, ShardSplitTest, RequiredFieldsTest, TermVectorComponentTest, TestComponentsName, TestDistributedSearch, SolrIndexSplitterTest, TestFunctionQuery, SolrTestCaseJ4Test, TermVectorComponentDistributedTest, SuggesterFSTTest, TestCSVLoader, TestFieldResource, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest]
[junit4:junit4] Completed in 66.26s, 1 test, 1 error <<< FAILURES!
[...truncated 445 lines...]
BUILD FAILED
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:385: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:365: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:39: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:181: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:437: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1240: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:884: There were test failures: 296 suites, 1243 tests, 1 suite-level error, 1 error, 18 ignored (6 assumptions)
Total time: 112 minutes 56 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure
[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.7.0) - Build # 564 -
Still Failing!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/564/
Java: 64bit/jdk1.7.0 -XX:+UseCompressedOops -XX:+UseSerialGC
1 tests failed.
FAILED: org.apache.solr.cloud.ShardSplitTest.testDistribSearch
Error Message:
Server at http://127.0.0.1:50269/js_y returned non ok status:500, message:Server Error
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:50269/js_y returned non ok status:500, message:Server Error
at __randomizedtesting.SeedInfo.seed([C15D2F8980D4126:8DF35CE0EF52211A]:0)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
at org.apache.solr.cloud.ShardSplitTest.splitShard(ShardSplitTest.java:228)
at org.apache.solr.cloud.ShardSplitTest.doTest(ShardSplitTest.java:150)
at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:601)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:722)
Build Log:
[...truncated 9499 lines...]
[junit4:junit4] Suite: org.apache.solr.cloud.ShardSplitTest
[junit4:junit4] 2> 1870713 T3907 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /js_y/
[junit4:junit4] 2> 1870732 T3907 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4:junit4] 2> Creating dataDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./solrtest-ShardSplitTest-1371296967097
[junit4:junit4] 2> 1870734 T3907 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4:junit4] 2> 1870735 T3908 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4:junit4] 2> 1870837 T3907 oasc.ZkTestServer.run start zk server on port:50262
[junit4:junit4] 2> 1870843 T3907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1870870 T3914 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62a04979 name:ZooKeeperConnection Watcher:127.0.0.1:50262 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1870871 T3907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1870871 T3907 oascc.SolrZkClient.makePath makePath: /solr
[junit4:junit4] 2> 1870888 T3907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1870909 T3916 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@119bea2c name:ZooKeeperConnection Watcher:127.0.0.1:50262/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1870910 T3907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1870910 T3907 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4:junit4] 2> 1870927 T3907 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4:junit4] 2> 1870936 T3907 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4:junit4] 2> 1870944 T3907 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4:junit4] 2> 1870953 T3907 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 1870955 T3907 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 1870969 T3907 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4:junit4] 2> 1870970 T3907 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4:junit4] 2> 1870980 T3907 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4:junit4] 2> 1870981 T3907 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4:junit4] 2> 1870990 T3907 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4:junit4] 2> 1870991 T3907 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4:junit4] 2> 1871000 T3907 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4:junit4] 2> 1871001 T3907 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4:junit4] 2> 1871008 T3907 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 1871010 T3907 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 1871024 T3907 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 1871026 T3907 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 1871035 T3907 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 1871036 T3907 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 1871046 T3907 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4:junit4] 2> 1871047 T3907 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4:junit4] 2> 1871898 T3907 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1871911 T3907 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50265
[junit4:junit4] 2> 1871913 T3907 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1871916 T3907 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1871917 T3907 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-controljetty-1371296967424
[junit4:junit4] 2> 1871917 T3907 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-controljetty-1371296967424/solr.xml
[junit4:junit4] 2> 1871918 T3907 oasc.CoreContainer.<init> New CoreContainer 416687493
[junit4:junit4] 2> 1871921 T3907 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-controljetty-1371296967424/'
[junit4:junit4] 2> 1871923 T3907 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-controljetty-1371296967424/'
[junit4:junit4] 2> 1872261 T3907 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1872262 T3907 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1872262 T3907 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1872263 T3907 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1872263 T3907 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1872264 T3907 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1872265 T3907 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1872266 T3907 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1872266 T3907 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1872267 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1872307 T3907 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1872314 T3907 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50262/solr
[junit4:junit4] 2> 1872318 T3907 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1872323 T3907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1872330 T3927 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3121cc9b name:ZooKeeperConnection Watcher:127.0.0.1:50262 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1872332 T3907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1872341 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1872383 T3907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1872389 T3929 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@8f61126 name:ZooKeeperConnection Watcher:127.0.0.1:50262/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1872389 T3907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1872398 T3907 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4:junit4] 2> 1872415 T3907 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4:junit4] 2> 1872429 T3907 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4:junit4] 2> 1872439 T3907 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50265_js_y
[junit4:junit4] 2> 1872448 T3907 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50265_js_y
[junit4:junit4] 2> 1872471 T3907 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4:junit4] 2> 1872496 T3907 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4:junit4] 2> 1872507 T3907 oasc.Overseer.start Overseer (id=89869318036520963-127.0.0.1:50265_js_y-n_0000000000) starting
[junit4:junit4] 2> 1872523 T3907 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4:junit4] 2> 1872550 T3931 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4:junit4] 2> 1872558 T3907 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4:junit4] 2> 1872571 T3907 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4:junit4] 2> 1872583 T3907 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1872600 T3930 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4:junit4] 2> 1872623 T3932 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1872626 T3932 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1874113 T3930 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1874115 T3930 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50265_js_y",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50265/js_y"}
[junit4:junit4] 2> 1874116 T3930 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
[junit4:junit4] 2> 1874116 T3930 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1874132 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1874635 T3932 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-controljetty-1371296967424/collection1
[junit4:junit4] 2> 1874635 T3932 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4:junit4] 2> 1874638 T3932 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1874639 T3932 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4:junit4] 2> 1874642 T3932 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-controljetty-1371296967424/collection1/'
[junit4:junit4] 2> 1874644 T3932 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-controljetty-1371296967424/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1874644 T3932 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-controljetty-1371296967424/collection1/lib/README' to classloader
[junit4:junit4] 2> 1874734 T3932 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1874840 T3932 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1874844 T3932 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1874850 T3932 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1875814 T3932 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1875821 T3932 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1875824 T3932 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1875834 T3932 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1875915 T3932 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 1875916 T3932 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-controljetty-1371296967424/collection1/, dataDir=./org.apache.solr.cloud.ShardSplitTest-1371296967096/control/data/
[junit4:junit4] 2> 1875916 T3932 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@35cc5477
[junit4:junit4] 2> 1875916 T3932 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 1875917 T3932 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371296967096/control/data
[junit4:junit4] 2> 1875918 T3932 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ShardSplitTest-1371296967096/control/data/index/
[junit4:junit4] 2> 1875918 T3932 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ShardSplitTest-1371296967096/control/data/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1875919 T3932 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371296967096/control/data/index
[junit4:junit4] 2> 1875926 T3932 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@641d1fa4 lockFactory=org.apache.lucene.store.NativeFSLockFactory@57d38293),segFN=segments_1,generation=1}
[junit4:junit4] 2> 1875926 T3932 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1875935 T3932 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1875938 T3932 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1875939 T3932 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1875940 T3932 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1875941 T3932 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1875941 T3932 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1875942 T3932 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1875942 T3932 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1875943 T3932 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1875966 T3932 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1875977 T3932 oass.SolrIndexSearcher.<init> Opening Searcher@5b3d4173 main
[junit4:junit4] 2> 1875978 T3932 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ShardSplitTest-1371296967096/control/data/tlog
[junit4:junit4] 2> 1875980 T3932 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1875980 T3932 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1875985 T3933 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5b3d4173 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1875988 T3932 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1875988 T3932 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50265/js_y collection:control_collection shard:shard1
[junit4:junit4] 2> 1876012 T3932 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4:junit4] 2> 1876039 T3932 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 1876046 T3932 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 1876047 T3932 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 1876047 T3932 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50265/js_y/collection1/
[junit4:junit4] 2> 1876047 T3932 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 1876048 T3932 oasc.SyncStrategy.syncToMe http://127.0.0.1:50265/js_y/collection1/ has no replicas
[junit4:junit4] 2> 1876048 T3932 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50265/js_y/collection1/
[junit4:junit4] 2> 1876048 T3932 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4:junit4] 2> 1877172 T3930 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1877199 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1877250 T3932 oasc.ZkController.register We are http://127.0.0.1:50265/js_y/collection1/ and leader is http://127.0.0.1:50265/js_y/collection1/
[junit4:junit4] 2> 1877251 T3932 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50265/js_y
[junit4:junit4] 2> 1877251 T3932 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 1877251 T3932 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1877251 T3932 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1877256 T3932 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1877258 T3907 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1877259 T3907 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1877259 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1877283 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:
[junit4:junit4] 2> 1877286 T3907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1877299 T3936 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ebcaafc name:ZooKeeperConnection Watcher:127.0.0.1:50262/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1877300 T3907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1877304 T3907 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1877316 T3907 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
[junit4:junit4] 2> 1877910 T3907 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1877918 T3907 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50269
[junit4:junit4] 2> 1877920 T3907 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1877920 T3907 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1877921 T3907 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty1-1371296973680
[junit4:junit4] 2> 1877921 T3907 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty1-1371296973680/solr.xml
[junit4:junit4] 2> 1877922 T3907 oasc.CoreContainer.<init> New CoreContainer 463919150
[junit4:junit4] 2> 1877923 T3907 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty1-1371296973680/'
[junit4:junit4] 2> 1877923 T3907 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty1-1371296973680/'
[junit4:junit4] 2> 1878145 T3907 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1878146 T3907 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1878147 T3907 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1878147 T3907 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1878148 T3907 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1878149 T3907 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1878149 T3907 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1878150 T3907 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1878150 T3907 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1878151 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1878188 T3907 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1878189 T3907 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50262/solr
[junit4:junit4] 2> 1878189 T3907 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1878192 T3907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1878199 T3947 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@61ad9d31 name:ZooKeeperConnection Watcher:127.0.0.1:50262 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1878200 T3907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1878207 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1878233 T3907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1878244 T3949 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@58ca2f73 name:ZooKeeperConnection Watcher:127.0.0.1:50262/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1878246 T3907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1878267 T3907 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1878719 T3930 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1878721 T3930 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50265_js_y",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50265/js_y"}
[junit4:junit4] 2> 1878733 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1878734 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1878733 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1879278 T3907 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50269_js_y
[junit4:junit4] 2> 1879282 T3907 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50269_js_y
[junit4:junit4] 2> 1879294 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1879308 T3929 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 1879309 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1879309 T3936 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 1879311 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1879313 T3949 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 1879319 T3950 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1879319 T3950 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1880260 T3930 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1880262 T3930 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50269_js_y",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50269/js_y"}
[junit4:junit4] 2> 1880262 T3930 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 2
[junit4:junit4] 2> 1880263 T3930 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1880274 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1880275 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1880275 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1880324 T3950 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty1-1371296973680/collection1
[junit4:junit4] 2> 1880324 T3950 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1880330 T3950 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1880331 T3950 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1880339 T3950 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty1-1371296973680/collection1/'
[junit4:junit4] 2> 1880340 T3950 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty1-1371296973680/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1880341 T3950 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty1-1371296973680/collection1/lib/README' to classloader
[junit4:junit4] 2> 1880432 T3950 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1880553 T3950 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1880557 T3950 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1880568 T3950 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1881487 T3950 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1881492 T3950 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1881496 T3950 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1881505 T3950 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1881583 T3950 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 1881584 T3950 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty1-1371296973680/collection1/, dataDir=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty1/
[junit4:junit4] 2> 1881584 T3950 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@35cc5477
[junit4:junit4] 2> 1881585 T3950 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 1881586 T3950 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty1
[junit4:junit4] 2> 1881586 T3950 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty1/index/
[junit4:junit4] 2> 1881587 T3950 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty1/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1881588 T3950 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty1/index
[junit4:junit4] 2> 1881593 T3950 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@54abe517 lockFactory=org.apache.lucene.store.NativeFSLockFactory@670c1068),segFN=segments_1,generation=1}
[junit4:junit4] 2> 1881594 T3950 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1881598 T3950 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1881599 T3950 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1881600 T3950 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1881602 T3950 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1881603 T3950 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1881603 T3950 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1881604 T3950 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1881605 T3950 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1881606 T3950 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1881635 T3950 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1881645 T3950 oass.SolrIndexSearcher.<init> Opening Searcher@1d259811 main
[junit4:junit4] 2> 1881662 T3950 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty1/tlog
[junit4:junit4] 2> 1881663 T3950 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1881664 T3950 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1881676 T3951 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1d259811 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1881679 T3950 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1881680 T3950 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50269/js_y collection:collection1 shard:shard1
[junit4:junit4] 2> 1881682 T3950 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4:junit4] 2> 1881703 T3950 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 1881710 T3950 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 1881711 T3950 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 1881711 T3950 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50269/js_y/collection1/
[junit4:junit4] 2> 1881711 T3950 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 1881712 T3950 oasc.SyncStrategy.syncToMe http://127.0.0.1:50269/js_y/collection1/ has no replicas
[junit4:junit4] 2> 1881712 T3950 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50269/js_y/collection1/
[junit4:junit4] 2> 1881712 T3950 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4:junit4] 2> 1881795 T3930 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1881820 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1881820 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1881820 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1881838 T3950 oasc.ZkController.register We are http://127.0.0.1:50269/js_y/collection1/ and leader is http://127.0.0.1:50269/js_y/collection1/
[junit4:junit4] 2> 1881839 T3950 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50269/js_y
[junit4:junit4] 2> 1881839 T3950 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 1881840 T3950 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1881840 T3950 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1881848 T3950 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1881853 T3907 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1881853 T3907 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1881854 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1882501 T3907 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1882511 T3907 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50272
[junit4:junit4] 2> 1882512 T3907 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1882512 T3907 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1882513 T3907 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty2-1371296978257
[junit4:junit4] 2> 1882514 T3907 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty2-1371296978257/solr.xml
[junit4:junit4] 2> 1882514 T3907 oasc.CoreContainer.<init> New CoreContainer 1662669952
[junit4:junit4] 2> 1882515 T3907 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty2-1371296978257/'
[junit4:junit4] 2> 1882516 T3907 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty2-1371296978257/'
[junit4:junit4] 2> 1882733 T3907 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1882734 T3907 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1882734 T3907 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1882735 T3907 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1882735 T3907 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1882736 T3907 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1882737 T3907 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1882737 T3907 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1882738 T3907 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1882738 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1882772 T3907 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1882774 T3907 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50262/solr
[junit4:junit4] 2> 1882774 T3907 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1882778 T3907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1882783 T3963 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@39aa1a2 name:ZooKeeperConnection Watcher:127.0.0.1:50262 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1882784 T3907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1882789 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1882816 T3907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1882821 T3965 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f402cba name:ZooKeeperConnection Watcher:127.0.0.1:50262/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1882822 T3907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1882837 T3907 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1883358 T3930 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1883360 T3930 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50269_js_y",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50269/js_y"}
[junit4:junit4] 2> 1883373 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1883378 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1883378 T3965 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1883380 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1883853 T3907 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50272_js_y
[junit4:junit4] 2> 1883862 T3907 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50272_js_y
[junit4:junit4] 2> 1883869 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1883870 T3965 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1883873 T3929 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1883874 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1883874 T3936 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1883875 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1883880 T3949 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1883884 T3965 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1883897 T3966 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1883897 T3966 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1884901 T3930 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1884903 T3930 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50272_js_y",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50272/js_y"}
[junit4:junit4] 2> 1884904 T3930 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 1884904 T3930 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 1884916 T3965 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1884917 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1884916 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1884917 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1885903 T3966 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty2-1371296978257/collection1
[junit4:junit4] 2> 1885904 T3966 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1885909 T3966 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1885910 T3966 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1885916 T3966 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty2-1371296978257/collection1/'
[junit4:junit4] 2> 1885918 T3966 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty2-1371296978257/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1885919 T3966 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty2-1371296978257/collection1/lib/README' to classloader
[junit4:junit4] 2> 1886069 T3966 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1886258 T3966 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1886262 T3966 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1886271 T3966 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1887373 T3966 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1887378 T3966 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1887381 T3966 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1887390 T3966 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1887471 T3966 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 1887472 T3966 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty2-1371296978257/collection1/, dataDir=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty2/
[junit4:junit4] 2> 1887472 T3966 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@35cc5477
[junit4:junit4] 2> 1887473 T3966 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 1887474 T3966 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty2
[junit4:junit4] 2> 1887475 T3966 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty2/index/
[junit4:junit4] 2> 1887475 T3966 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty2/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1887478 T3966 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty2/index
[junit4:junit4] 2> 1887490 T3966 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@6f520559 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7eda960a),segFN=segments_1,generation=1}
[junit4:junit4] 2> 1887490 T3966 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1887497 T3966 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1887500 T3966 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1887519 T3966 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1887520 T3966 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1887521 T3966 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1887522 T3966 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1887523 T3966 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1887525 T3966 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1887526 T3966 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1887558 T3966 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1887573 T3966 oass.SolrIndexSearcher.<init> Opening Searcher@1f6e7f7d main
[junit4:junit4] 2> 1887574 T3966 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty2/tlog
[junit4:junit4] 2> 1887578 T3966 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1887578 T3966 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1887587 T3967 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1f6e7f7d main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1887591 T3966 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1887591 T3966 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50272/js_y collection:collection1 shard:shard2
[junit4:junit4] 2> 1887593 T3966 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4:junit4] 2> 1887617 T3966 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 1887625 T3966 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 1887625 T3966 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 1887626 T3966 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50272/js_y/collection1/
[junit4:junit4] 2> 1887627 T3966 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 1887628 T3966 oasc.SyncStrategy.syncToMe http://127.0.0.1:50272/js_y/collection1/ has no replicas
[junit4:junit4] 2> 1887628 T3966 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50272/js_y/collection1/
[junit4:junit4] 2> 1887628 T3966 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4:junit4] 2> 1887950 T3930 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1887970 T3965 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1887971 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1887971 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1887971 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1888004 T3966 oasc.ZkController.register We are http://127.0.0.1:50272/js_y/collection1/ and leader is http://127.0.0.1:50272/js_y/collection1/
[junit4:junit4] 2> 1888005 T3966 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50272/js_y
[junit4:junit4] 2> 1888005 T3966 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 1888005 T3966 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1888006 T3966 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1888010 T3966 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1888014 T3907 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1888015 T3907 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1888016 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1888667 T3907 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1888681 T3907 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50275
[junit4:junit4] 2> 1888683 T3907 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1888684 T3907 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1888685 T3907 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty3-1371296984413
[junit4:junit4] 2> 1888685 T3907 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty3-1371296984413/solr.xml
[junit4:junit4] 2> 1888686 T3907 oasc.CoreContainer.<init> New CoreContainer 286124825
[junit4:junit4] 2> 1888687 T3907 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty3-1371296984413/'
[junit4:junit4] 2> 1888688 T3907 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty3-1371296984413/'
[junit4:junit4] 2> 1888933 T3907 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1888938 T3907 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1888939 T3907 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1888939 T3907 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1888941 T3907 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1888942 T3907 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1888942 T3907 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1888943 T3907 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1888943 T3907 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1888945 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1888974 T3907 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1888975 T3907 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50262/solr
[junit4:junit4] 2> 1888976 T3907 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1888979 T3907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1888985 T3979 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1d226030 name:ZooKeeperConnection Watcher:127.0.0.1:50262 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1888986 T3907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1889000 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1889026 T3907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1889032 T3981 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@75e3d46e name:ZooKeeperConnection Watcher:127.0.0.1:50262/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1889033 T3907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1889046 T3907 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1889501 T3930 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1889503 T3930 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50272_js_y",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50272/js_y"}
[junit4:junit4] 2> 1889515 T3981 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1889515 T3965 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1889515 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1889516 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1889515 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1890058 T3907 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50275_js_y
[junit4:junit4] 2> 1890060 T3907 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50275_js_y
[junit4:junit4] 2> 1890068 T3981 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1890072 T3929 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1890072 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1890076 T3965 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1890077 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1890080 T3981 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1890097 T3982 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1890099 T3982 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1890103 T3936 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1890103 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1890107 T3965 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1890108 T3949 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1891035 T3930 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1891038 T3930 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50275_js_y",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50275/js_y"}
[junit4:junit4] 2> 1891038 T3930 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 1891038 T3930 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1891054 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1891054 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1891055 T3965 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1891056 T3981 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1891056 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1891107 T3982 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty3-1371296984413/collection1
[junit4:junit4] 2> 1891108 T3982 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1891110 T3982 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1891110 T3982 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1891116 T3982 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty3-1371296984413/collection1/'
[junit4:junit4] 2> 1891118 T3982 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty3-1371296984413/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1891118 T3982 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty3-1371296984413/collection1/lib/README' to classloader
[junit4:junit4] 2> 1891240 T3982 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1891354 T3982 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1891358 T3982 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1891375 T3982 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1892450 T3982 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1892494 T3982 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1892507 T3982 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1892517 T3982 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1892617 T3982 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 1892618 T3982 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty3-1371296984413/collection1/, dataDir=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty3/
[junit4:junit4] 2> 1892618 T3982 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@35cc5477
[junit4:junit4] 2> 1892619 T3982 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 1892621 T3982 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty3
[junit4:junit4] 2> 1892621 T3982 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty3/index/
[junit4:junit4] 2> 1892622 T3982 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty3/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1892623 T3982 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty3/index
[junit4:junit4] 2> 1892628 T3982 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@39750675 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7d964cea),segFN=segments_1,generation=1}
[junit4:junit4] 2> 1892629 T3982 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1892634 T3982 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1892634 T3982 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1892635 T3982 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1892636 T3982 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1892638 T3982 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1892638 T3982 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1892642 T3982 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1892646 T3982 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1892648 T3982 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1892676 T3982 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1892690 T3982 oass.SolrIndexSearcher.<init> Opening Searcher@79faaa88 main
[junit4:junit4] 2> 1892691 T3982 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty3/tlog
[junit4:junit4] 2> 1892692 T3982 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1892693 T3982 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1892699 T3983 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@79faaa88 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1892710 T3982 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1892710 T3982 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50275/js_y collection:collection1 shard:shard1
[junit4:junit4] 2> 1892722 T3982 oasc.ZkController.register We are http://127.0.0.1:50275/js_y/collection1/ and leader is http://127.0.0.1:50269/js_y/collection1/
[junit4:junit4] 2> 1892723 T3982 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50275/js_y
[junit4:junit4] 2> 1892723 T3982 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 1892723 T3982 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C2918 name=collection1 org.apache.solr.core.SolrCore@10d64873 url=http://127.0.0.1:50275/js_y/collection1 node=127.0.0.1:50275_js_y C2918_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:50275_js_y, base_url=http://127.0.0.1:50275/js_y}
[junit4:junit4] 2> 1892725 T3984 C2918 P50275 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 1892726 T3984 C2918 P50275 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 1892726 T3982 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1892726 T3984 C2918 P50275 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 1892727 T3984 C2918 P50275 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1892730 T3907 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1892731 T3907 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1892731 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1892745 T3984 C2918 P50275 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1892782 T3945 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 1893584 T3907 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1893595 T3907 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50279
[junit4:junit4] 2> 1893596 T3907 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1893597 T3907 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1893597 T3907 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty4-1371296989125
[junit4:junit4] 2> 1893598 T3907 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty4-1371296989125/solr.xml
[junit4:junit4] 2> 1893598 T3907 oasc.CoreContainer.<init> New CoreContainer 1732343409
[junit4:junit4] 2> 1893600 T3907 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty4-1371296989125/'
[junit4:junit4] 2> 1893600 T3907 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty4-1371296989125/'
[junit4:junit4] 2> 1893818 T3907 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1893819 T3907 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1893820 T3907 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1893820 T3907 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1893821 T3907 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1893822 T3907 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1893822 T3907 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1893823 T3907 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1893823 T3907 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1893824 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1893863 T3907 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1893864 T3907 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50262/solr
[junit4:junit4] 2> 1893865 T3907 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1893867 T3907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1893874 T3996 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@114153be name:ZooKeeperConnection Watcher:127.0.0.1:50262 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1893875 T3907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1893882 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1893920 T3907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1893926 T3998 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6b102f1c name:ZooKeeperConnection Watcher:127.0.0.1:50262/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1893926 T3907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1893944 T3907 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1894093 T3930 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1894096 T3930 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50275_js_y",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50275/js_y"}
[junit4:junit4] 2> 1894108 T3981 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1894108 T3998 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1894109 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1894109 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1894109 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1894109 T3965 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1894788 T3945 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true for: 2 seconds.
[junit4:junit4] 2> 1894789 T3945 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=4&state=recovering&nodeName=127.0.0.1:50275_js_y&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=2007
[junit4:junit4] 2> 1894958 T3907 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50279_js_y
[junit4:junit4] 2> 1894963 T3907 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50279_js_y
[junit4:junit4] 2> 1894972 T3965 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1894974 T3981 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1894976 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1894981 T3936 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1894981 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1894982 T3998 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1894982 T3998 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1894983 T3929 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1894983 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1894997 T3949 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1894998 T3981 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1894997 T3965 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1895026 T3999 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1895026 T3999 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1895633 T3930 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1895636 T3930 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50279_js_y",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50279/js_y"}
[junit4:junit4] 2> 1895636 T3930 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 1895637 T3930 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 1895652 T3998 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1895652 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1895653 T3981 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1895653 T3965 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1895654 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1895655 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1896031 T3999 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty4-1371296989125/collection1
[junit4:junit4] 2> 1896031 T3999 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1896038 T3999 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1896039 T3999 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1896043 T3999 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty4-1371296989125/collection1/'
[junit4:junit4] 2> 1896044 T3999 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty4-1371296989125/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1896045 T3999 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ShardSplitTest-jetty4-1371296989125/collection1/lib/README' to classloader
[junit4:junit4] 2> 1896154 T3999 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1896260 T3999 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1896264 T3999 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1896271 T3999 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> ASYNC NEW_CORE C2919 name=collection1 org.apache.solr.core.SolrCore@10d64873 url=http://127.0.0.1:50275/js_y/collection1 node=127.0.0.1:50275_js_y C2919_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:50275_js_y, base_url=http://127.0.0.1:50275/js_y}
[junit4:junit4] 2> 1896793 T3984 C2919 P50275 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:50269/js_y/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 1896793 T3984 C2919 P50275 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:50275/js_y START replicas=[http://127.0.0.1:50269/js_y/collection1/] nUpdates=100
[junit4:junit4] 2> 1896794 T3984 C2919 P50275 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 1896796 T3984 C2919 P50275 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 1896798 T3984 C2919 P50275 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 1896798 T3984 C2919 P50275 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 1896799 T3984 C2919 P50275 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 1896799 T3984 C2919 P50275 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:50269/js_y/collection1/. core=collection1
[junit4:junit4] 2> 1896799 T3984 C2919 P50275 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C2920 name=collection1 org.apache.solr.core.SolrCore@719c8968 url=http://127.0.0.1:50269/js_y/collection1 node=127.0.0.1:50269_js_y C2920_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50269_js_y, base_url=http://127.0.0.1:50269/js_y, leader=true}
[junit4:junit4] 2> 1896810 T3945 C2920 P50269 oasc.SolrCore.execute [collection1] webapp=/js_y path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=1
[junit4:junit4] 2> 1896873 T3942 C2920 P50269 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 1896888 T3942 C2920 P50269 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@54abe517 lockFactory=org.apache.lucene.store.NativeFSLockFactory@670c1068),segFN=segments_1,generation=1}
[junit4:junit4] 2> 1896909 T3942 C2920 P50269 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1896911 T3942 C2920 P50269 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@54abe517 lockFactory=org.apache.lucene.store.NativeFSLockFactory@670c1068),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@54abe517 lockFactory=org.apache.lucene.store.NativeFSLockFactory@670c1068),segFN=segments_2,generation=2}
[junit4:junit4] 2> 1896912 T3942 C2920 P50269 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1896912 T3942 C2920 P50269 oass.SolrIndexSearcher.<init> Opening Searcher@7ed4aec3 realtime
[junit4:junit4] 2> 1896913 T3942 C2920 P50269 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1896913 T3942 C2920 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 40
[junit4:junit4] 2> 1896918 T3984 C2919 P50275 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1896919 T3984 C2919 P50275 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 1896931 T3944 C2920 P50269 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1896932 T3944 C2920 P50269 oasc.SolrCore.execute [collection1] webapp=/js_y path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=8
[junit4:junit4] 2> 1896938 T3984 C2919 P50275 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 1896938 T3984 C2919 P50275 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 1896938 T3984 C2919 P50275 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 1896965 T3945 C2920 P50269 oasc.SolrCore.execute [collection1] webapp=/js_y path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=13
[junit4:junit4] 2> 1896967 T3984 C2919 P50275 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 1896983 T3984 C2919 P50275 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty3/index.20130615154953331
[junit4:junit4] 2> 1896984 T3984 C2919 P50275 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@23e87ead lockFactory=org.apache.lucene.store.NativeFSLockFactory@2dfc8289) fullCopy=false
[junit4:junit4] 2> 1896999 T3940 C2920 P50269 oasc.SolrCore.execute [collection1] webapp=/js_y path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0
[junit4:junit4] 2> 1897017 T3984 C2919 P50275 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 1897019 T3984 C2919 P50275 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 1897020 T3984 C2919 P50275 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 1897023 T3984 C2919 P50275 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@39750675 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7d964cea),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@39750675 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7d964cea),segFN=segments_2,generation=2}
[junit4:junit4] 2> 1897024 T3984 C2919 P50275 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1897024 T3984 C2919 P50275 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 1897025 T3984 C2919 P50275 oass.SolrIndexSearcher.<init> Opening Searcher@53f97013 main
[junit4:junit4] 2> 1897026 T3983 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@53f97013 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 1897029 T3984 C2919 P50275 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty3/index.20130615154953331 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty3/index.20130615154953331;done=true>>]
[junit4:junit4] 2> 1897029 T3984 C2919 P50275 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty3/index.20130615154953331
[junit4:junit4] 2> 1897029 T3984 C2919 P50275 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty3/index.20130615154953331
[junit4:junit4] 2> 1897030 T3984 C2919 P50275 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 1897030 T3984 C2919 P50275 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 1897030 T3984 C2919 P50275 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1897031 T3984 C2919 P50275 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1897035 T3984 C2919 P50275 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 1897172 T3930 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1897174 T3930 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50275_js_y",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50275/js_y"}
[junit4:junit4] 2> 1897190 T3981 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1897190 T3965 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1897190 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1897191 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1897191 T3998 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1897192 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1897545 T3999 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1897551 T3999 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1897553 T3999 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1897563 T3999 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1897639 T3999 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 1897639 T3999 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-jetty4-1371296989125/collection1/, dataDir=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4/
[junit4:junit4] 2> 1897640 T3999 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@35cc5477
[junit4:junit4] 2> 1897641 T3999 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 1897642 T3999 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4
[junit4:junit4] 2> 1897642 T3999 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4/index/
[junit4:junit4] 2> 1897643 T3999 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1897644 T3999 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4/index
[junit4:junit4] 2> 1897651 T3999 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@50f9409a lockFactory=org.apache.lucene.store.NativeFSLockFactory@45f45cb0),segFN=segments_1,generation=1}
[junit4:junit4] 2> 1897651 T3999 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1897655 T3999 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1897658 T3999 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1897659 T3999 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1897664 T3999 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1897665 T3999 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1897666 T3999 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1897667 T3999 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1897668 T3999 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1897670 T3999 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1897695 T3999 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1897706 T3999 oass.SolrIndexSearcher.<init> Opening Searcher@6e8cb455 main
[junit4:junit4] 2> 1897706 T3999 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4/tlog
[junit4:junit4] 2> 1897708 T3999 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1897708 T3999 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1897714 T4002 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6e8cb455 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1897719 T3999 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1897719 T3999 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50279/js_y collection:collection1 shard:shard2
[junit4:junit4] 2> 1897734 T3999 oasc.ZkController.register We are http://127.0.0.1:50279/js_y/collection1/ and leader is http://127.0.0.1:50272/js_y/collection1/
[junit4:junit4] 2> 1897735 T3999 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50279/js_y
[junit4:junit4] 2> 1897735 T3999 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 1897735 T3999 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C2921 name=collection1 org.apache.solr.core.SolrCore@4b24cc95 url=http://127.0.0.1:50279/js_y/collection1 node=127.0.0.1:50279_js_y C2921_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:50279_js_y, base_url=http://127.0.0.1:50279/js_y}
[junit4:junit4] 2> 1897736 T4003 C2921 P50279 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 1897738 T4003 C2921 P50279 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 1897738 T4003 C2921 P50279 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 1897738 T3999 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1897738 T4003 C2921 P50279 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1897744 T3907 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1897744 T3907 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1897744 T4003 C2921 P50279 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1897745 T3907 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1897786 T3907 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1897787 T3957 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 1897789 T3907 oasc.AbstractFullDistribZkTestBase.waitForThingsToLevelOut Wait for recoveries to finish - wait 15 for each attempt
[junit4:junit4] 2> 1897790 T3907 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):15
[junit4:junit4] 2> 1897792 T3907 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1898720 T3930 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1898722 T3930 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50279_js_y",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50279/js_y"}
[junit4:junit4] 2> 1898740 T3981 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1898740 T3998 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1898740 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1898741 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1898740 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1898740 T3965 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1898790 T3957 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 1898793 T3957 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=5&state=recovering&nodeName=127.0.0.1:50279_js_y&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1006
[junit4:junit4] 2> 1898798 T3907 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1899803 T3907 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> C2921_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:50279_js_y, base_url=http://127.0.0.1:50279/js_y}
[junit4:junit4] 2> 1900798 T4003 C2921 P50279 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:50272/js_y/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 1900800 T4003 C2921 P50279 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:50279/js_y START replicas=[http://127.0.0.1:50272/js_y/collection1/] nUpdates=100
[junit4:junit4] 2> 1900801 T4003 C2921 P50279 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 1900801 T4003 C2921 P50279 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 1900803 T4003 C2921 P50279 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 1900805 T4003 C2921 P50279 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 1900805 T4003 C2921 P50279 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 1900806 T4003 C2921 P50279 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:50272/js_y/collection1/. core=collection1
[junit4:junit4] 2> 1900806 T4003 C2921 P50279 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1900810 T3907 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> ASYNC NEW_CORE C2922 name=collection1 org.apache.solr.core.SolrCore@6f1ce2c9 url=http://127.0.0.1:50272/js_y/collection1 node=127.0.0.1:50272_js_y C2922_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50272_js_y, base_url=http://127.0.0.1:50272/js_y, leader=true}
[junit4:junit4] 2> 1900846 T3957 C2922 P50272 oasc.SolrCore.execute [collection1] webapp=/js_y path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=13
[junit4:junit4] 2> 1900851 T3961 C2922 P50272 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 1900860 T3961 C2922 P50272 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@6f520559 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7eda960a),segFN=segments_1,generation=1}
[junit4:junit4] 2> 1900864 T3961 C2922 P50272 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1900865 T3961 C2922 P50272 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@6f520559 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7eda960a),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@6f520559 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7eda960a),segFN=segments_2,generation=2}
[junit4:junit4] 2> 1900866 T3961 C2922 P50272 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1900867 T3961 C2922 P50272 oass.SolrIndexSearcher.<init> Opening Searcher@7aea369 realtime
[junit4:junit4] 2> 1900867 T3961 C2922 P50272 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1900868 T3961 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 17
[junit4:junit4] 2> 1900870 T4003 C2921 P50279 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1900870 T4003 C2921 P50279 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 1900874 T3958 C2922 P50272 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1900875 T3958 C2922 P50272 oasc.SolrCore.execute [collection1] webapp=/js_y path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=2
[junit4:junit4] 2> 1900876 T4003 C2921 P50279 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 1900876 T4003 C2921 P50279 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 1900876 T4003 C2921 P50279 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 1900880 T3959 C2922 P50272 oasc.SolrCore.execute [collection1] webapp=/js_y path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=1
[junit4:junit4] 2> 1900881 T4003 C2921 P50279 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 1900883 T4003 C2921 P50279 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4/index.20130615154957245
[junit4:junit4] 2> 1900884 T4003 C2921 P50279 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@7a97f9a5 lockFactory=org.apache.lucene.store.NativeFSLockFactory@38225eec) fullCopy=false
[junit4:junit4] 2> 1900887 T3960 C2922 P50272 oasc.SolrCore.execute [collection1] webapp=/js_y path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0
[junit4:junit4] 2> 1900891 T4003 C2921 P50279 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 1900893 T4003 C2921 P50279 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 1900893 T4003 C2921 P50279 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 1900896 T4003 C2921 P50279 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@50f9409a lockFactory=org.apache.lucene.store.NativeFSLockFactory@45f45cb0),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@50f9409a lockFactory=org.apache.lucene.store.NativeFSLockFactory@45f45cb0),segFN=segments_2,generation=2}
[junit4:junit4] 2> 1900898 T4003 C2921 P50279 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1900898 T4003 C2921 P50279 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 1900899 T4003 C2921 P50279 oass.SolrIndexSearcher.<init> Opening Searcher@330c80bd main
[junit4:junit4] 2> 1900901 T4002 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@330c80bd main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 1900902 T4003 C2921 P50279 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4/index.20130615154957245 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4/index.20130615154957245;done=true>>]
[junit4:junit4] 2> 1900902 T4003 C2921 P50279 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4/index.20130615154957245
[junit4:junit4] 2> 1900903 T4003 C2921 P50279 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4/index.20130615154957245
[junit4:junit4] 2> 1900903 T4003 C2921 P50279 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 1900904 T4003 C2921 P50279 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 1900904 T4003 C2921 P50279 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1900904 T4003 C2921 P50279 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1900908 T4003 C2921 P50279 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 1901778 T3930 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1901780 T3930 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50279_js_y",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50279/js_y"}
[junit4:junit4] 2> 1901793 T3981 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1901793 T3936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1901794 T3929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1901797 T3965 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1901798 T3949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1901799 T3998 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1901815 T3907 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1901821 T3907 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Recoveries finished - collection: collection1
[junit4:junit4] 2> ASYNC NEW_CORE C2923 name=collection1 org.apache.solr.core.SolrCore@7a633564 url=http://127.0.0.1:50265/js_y/collection1 node=127.0.0.1:50265_js_y C2923_STATE=coll:control_collection core:collection1 props:{shard=shard1, state=active, core=collection1, collection=control_collection, node_name=127.0.0.1:50265_js_y, base_url=http://127.0.0.1:50265/js_y, leader=true}
[junit4:junit4] 2> 1901836 T3922 C2923 P50265 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 1901839 T3922 C2923 P50265 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@641d1fa4 lockFactory=org.apache.lucene.store.NativeFSLockFactory@57d38293),segFN=segments_1,generation=1}
[junit4:junit4] 2> 1901841 T3922 C2923 P50265 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1901842 T3922 C2923 P50265 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@641d1fa4 lockFactory=org.apache.lucene.store.NativeFSLockFactory@57d38293),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@641d1fa4 lockFactory=org.apache.lucene.store.NativeFSLockFactory@57d38293),segFN=segments_2,generation=2}
[junit4:junit4] 2> 1901843 T3922 C2923 P50265 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1901844 T3922 C2923 P50265 oass.SolrIndexSearcher.<init> Opening Searcher@1163094f main
[junit4:junit4] 2> 1901844 T3922 C2923 P50265 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1901849 T3933 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1163094f main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 1901854 T3922 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 18
[junit4:junit4] 2> 1901890 T3956 C2922 P50272 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 1901892 T3956 C2922 P50272 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@6f520559 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7eda960a),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@6f520559 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7eda960a),segFN=segments_3,generation=3}
[junit4:junit4] 2> 1901893 T3956 C2922 P50272 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 1901894 T3956 C2922 P50272 oass.SolrIndexSearcher.<init> Opening Searcher@276d835d main
[junit4:junit4] 2> 1901895 T3956 C2922 P50272 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1901896 T3967 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@276d835d main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 1901896 T3956 C2922 P50272 oasu.SolrCmdDistributor.distribCommit Distrib commit to:[StdNode: http://127.0.0.1:50269/js_y/collection1/, StdNode: http://127.0.0.1:50275/js_y/collection1/, StdNode: http://127.0.0.1:50279/js_y/collection1/] params:commit_end_point=true&commit=true&softCommit=false&waitSearcher=true&expungeDeletes=false
[junit4:junit4] 2> ASYNC NEW_CORE C2924 name=collection1 org.apache.solr.core.SolrCore@719c8968 url=http://127.0.0.1:50269/js_y/collection1 node=127.0.0.1:50269_js_y C2924_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50269_js_y, base_url=http://127.0.0.1:50269/js_y, leader=true}
[junit4:junit4] 2> 1901907 T3941 C2924 P50269 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 1901919 T3941 C2924 P50269 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@54abe517 lockFactory=org.apache.lucene.store.NativeFSLockFactory@670c1068),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@54abe517 lockFactory=org.apache.lucene.store.NativeFSLockFactory@670c1068),segFN=segments_3,generation=3}
[junit4:junit4] 2> 1901920 T3941 C2924 P50269 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 1901920 T3941 C2924 P50269 oass.SolrIndexSearcher.<init> Opening Searcher@33f41a85 main
[junit4:junit4] 2> 1901921 T3941 C2924 P50269 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1901922 T3951 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@33f41a85 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 1901923 T3941 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 16
[junit4:junit4] 2> ASYNC NEW_CORE C2925 name=collection1 org.apache.solr.core.SolrCore@10d64873 url=http://127.0.0.1:50275/js_y/collection1 node=127.0.0.1:50275_js_y C2925_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50275_js_y, base_url=http://127.0.0.1:50275/js_y}
[junit4:junit4] 2> 1901927 T3973 C2925 P50275 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> C2921_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50279_js_y, base_url=http://127.0.0.1:50279/js_y}
[junit4:junit4] 2> 1901919 T3991 C2921 P50279 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 1901934 T3973 C2925 P50275 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@39750675 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7d964cea),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@39750675 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7d964cea),segFN=segments_3,generation=3}
[junit4:junit4] 2> 1901934 T3973 C2925 P50275 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 1901936 T3991 C2921 P50279 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@50f9409a lockFactory=org.apache.lucene.store.NativeFSLockFactory@45f45cb0),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@50f9409a lockFactory=org.apache.lucene.store.NativeFSLockFactory@45f45cb0),segFN=segments_3,generation=3}
[junit4:junit4] 2> 1901936 T3991 C2921 P50279 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 1901937 T3973 C2925 P50275 oass.SolrIndexSearcher.<init> Opening Searcher@674b1239 main
[junit4:junit4] 2> 1901937 T3973 C2925 P50275 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1901938 T3983 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@674b1239 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 1901939 T3973 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 12
[junit4:junit4] 2> 1901943 T3991 C2921 P50279 oass.SolrIndexSearcher.<init> Opening Searcher@19095503 main
[junit4:junit4] 2> 1901944 T3991 C2921 P50279 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1901947 T4002 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@19095503 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 1901948 T3991 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 29
[junit4:junit4] 2> 1901950 T3956 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 60
[junit4:junit4] 2> 1901957 T3907 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1901967 T3942 C2924 P50269 oasc.SolrCore.execute [collection1] webapp=/js_y path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=5
[junit4:junit4] 2> 1901980 T3975 C2925 P50275 oasc.SolrCore.execute [collection1] webapp=/js_y path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 1901984 T3958 C2922 P50272 oasc.SolrCore.execute [collection1] webapp=/js_y path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 1901993 T3992 C2921 P50279 oasc.SolrCore.execute [collection1] webapp=/js_y path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 1904008 T3922 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1437909123456827392)} 0 6
[junit4:junit4] 2> 1904039 T3975 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={update.distrib=FROMLEADER&_version_=-1437909123474653184&update.from=http://127.0.0.1:50269/js_y/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1437909123474653184)} 0 2
[junit4:junit4] 2> 1904046 T3992 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={update.distrib=FROMLEADER&_version_=-1437909123492478976&update.from=http://127.0.0.1:50272/js_y/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1437909123492478976)} 0 2
[junit4:junit4] 2> 1904047 T3958 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={update.distrib=TOLEADER&wt=javabin&version=2} {deleteByQuery=*:* (-1437909123492478976)} 0 10
[junit4:junit4] 2> 1904049 T3942 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1437909123474653184)} 0 29
[junit4:junit4] 2> 1904054 T3924 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[a!0 (1437909123509256192)]} 0 1
[junit4:junit4] 2> 1904064 T3993 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[a!0 (1437909123517644800)]} 0 0
[junit4:junit4] 2> 1904066 T3959 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[a!0 (1437909123517644800)]} 0 6
[junit4:junit4] 2> 1904068 T3943 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[a!0]} 0 11
[junit4:junit4] 2> 1904071 T3923 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[b!1 (1437909123528130560)]} 0 0
[junit4:junit4] 2> 1904079 T3976 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[b!1 (1437909123531276288)]} 0 1
[junit4:junit4] 2> 1904080 T3944 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[b!1 (1437909123531276288)]} 0 6
[junit4:junit4] 2> 1904084 T3925 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[c!2 (1437909123540713472)]} 0 1
[junit4:junit4] 2> 1904098 T3977 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[c!2 (1437909123553296384)]} 0 0
[junit4:junit4] 2> 1904100 T3945 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[c!2 (1437909123553296384)]} 0 6
[junit4:junit4] 2> 1904101 T3960 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[c!2]} 0 13
[junit4:junit4] 2> 1904108 T3920 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[d!3 (1437909123565879296)]} 0 1
[junit4:junit4] 2> 1904117 T3994 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[d!3 (1437909123573219328)]} 0 1
[junit4:junit4] 2> 1904118 T3957 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[d!3 (1437909123573219328)]} 0 4
[junit4:junit4] 2> 1904119 T3940 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[d!3]} 0 8
[junit4:junit4] 2> 1904123 T3921 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[e!4 (1437909123582656512)]} 0 0
[junit4:junit4] 2> 1904134 T3989 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[e!4 (1437909123589996544)]} 0 1
[junit4:junit4] 2> 1904136 T3961 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[e!4 (1437909123589996544)]} 0 7
[junit4:junit4] 2> 1904138 T3941 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[e!4]} 0 11
[junit4:junit4] 2> 1904142 T3922 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[f!5 (1437909123601530880)]} 0 1
[junit4:junit4] 2> 1904151 T3990 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[f!5 (1437909123607822336)]} 0 1
[junit4:junit4] 2> 1904153 T3956 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[f!5 (1437909123607822336)]} 0 6
[junit4:junit4] 2> 1904159 T3942 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[f!5]} 0 14
[junit4:junit4] 2> 1904164 T3924 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[g!6 (1437909123625648128)]} 0 1
[junit4:junit4] 2> 1904176 T3972 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[g!6 (1437909123634036736)]} 0 1
[junit4:junit4] 2> 1904177 T3943 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[g!6 (1437909123634036736)]} 0 5
[junit4:junit4] 2> 1904181 T3923 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[h!7 (1437909123643473920)]} 0 0
[junit4:junit4] 2> 1904191 T3973 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[h!7 (1437909123648716800)]} 0 1
[junit4:junit4] 2> 1904193 T3944 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[h!7 (1437909123648716800)]} 0 7
[junit4:junit4] 2> 1904198 T3925 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[i!8 (1437909123661299712)]} 0 0
[junit4:junit4] 2> 1904205 T3974 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[i!8 (1437909123664445440)]} 0 1
[junit4:junit4] 2> 1904206 T3945 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[i!8 (1437909123664445440)]} 0 5
[junit4:junit4] 2> 1904210 T3920 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[j!9 (1437909123672834048)]} 0 1
[junit4:junit4] 2> 1904216 T3975 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[j!9 (1437909123677028352)]} 0 1
[junit4:junit4] 2> 1904217 T3940 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[j!9 (1437909123677028352)]} 0 5
[junit4:junit4] 2> 1904226 T3921 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[k!10 (1437909123687514112)]} 0 3
[junit4:junit4] 2> 1904242 T3976 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[k!10 (1437909123704291328)]} 0 0
[junit4:junit4] 2> 1904244 T3941 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[k!10 (1437909123704291328)]} 0 6
[junit4:junit4] 2> 1904249 T3922 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[l!11 (1437909123713728512)]} 0 2
[junit4:junit4] 2> 1904258 T3991 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[l!11 (1437909123718971392)]} 0 1
[junit4:junit4] 2> 1904260 T3958 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[l!11 (1437909123718971392)]} 0 7
[junit4:junit4] 2> 1904279 T3924 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[m!12 (1437909123745185792)]} 0 1
[junit4:junit4] 2> 1904286 T3992 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[m!12 (1437909123749380096)]} 0 1
[junit4:junit4] 2> 1904287 T3959 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[m!12 (1437909123749380096)]} 0 5
[junit4:junit4] 2> 1904290 T3925 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[n!13 (1437909123757768704)]} 0 0
[junit4:junit4] 2> 1904306 T3977 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[n!13 (1437909123768254464)]} 0 0
[junit4:junit4] 2> 1904307 T3942 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[n!13 (1437909123768254464)]} 0 9
[junit4:junit4] 2> 1904308 T3960 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[n!13]} 0 14
[junit4:junit4] 2> 1904312 T3920 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[o!14 (1437909123780837376)]} 0 0
[junit4:junit4] 2> 1904324 T3993 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[o!14 (1437909123789225984)]} 0 1
[junit4:junit4] 2> 1904326 T3957 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[o!14 (1437909123789225984)]} 0 6
[junit4:junit4] 2> 1904328 T3943 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[o!14]} 0 10
[junit4:junit4] 2> 1904333 T3921 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[p!15 (1437909123801808896)]} 0 1
[junit4:junit4] 2> 1904342 T3972 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[p!15 (1437909123809148928)]} 0 1
[junit4:junit4] 2> 1904343 T3944 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[p!15 (1437909123809148928)]} 0 4
[junit4:junit4] 2> 1904345 T3961 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[p!15]} 0 9
[junit4:junit4] 2> 1904348 T3922 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[q!16 (1437909123818586112)]} 0 0
[junit4:junit4] 2> 1904357 T3973 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[q!16 (1437909123824877568)]} 0 0
[junit4:junit4] 2> 1904358 T3945 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[q!16 (1437909123824877568)]} 0 4
[junit4:junit4] 2> 1904362 T3956 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[q!16]} 0 11
[junit4:junit4] 2> 1904371 T3924 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[r!17 (1437909123841654784)]} 0 1
[junit4:junit4] 2> 1904378 T3994 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[r!17 (1437909123845849088)]} 0 1
[junit4:junit4] 2> 1904379 T3958 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[r!17 (1437909123845849088)]} 0 5
[junit4:junit4] 2> 1904382 T3923 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[s!18 (1437909123854237696)]} 0 0
[junit4:junit4] 2> 1904391 T3974 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[s!18 (1437909123859480576)]} 0 1
[junit4:junit4] 2> 1904392 T3940 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[s!18 (1437909123859480576)]} 0 5
[junit4:junit4] 2> 1904397 T3925 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[t!19 (1437909123869966336)]} 0 1
[junit4:junit4] 2> 1904403 T3976 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[t!19 (1437909123873112064)]} 0 0
[junit4:junit4] 2> 1904404 T3942 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[t!19 (1437909123873112064)]} 0 4
[junit4:junit4] 2> 1904408 T3920 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[u!20 (1437909123881500672)]} 0 0
[junit4:junit4] 2> 1904417 T3989 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[u!20 (1437909123887792128)]} 0 0
[junit4:junit4] 2> 1904418 T3959 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[u!20 (1437909123887792128)]} 0 4
[junit4:junit4] 2> 1904421 T3943 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[u!20]} 0 9
[junit4:junit4] 2> 1904430 T3921 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[v!21 (1437909123903520768)]} 0 1
[junit4:junit4] 2> 1904442 T3977 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[v!21 (1437909123909812224)]} 0 1
[junit4:junit4] 2> 1904443 T3944 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[v!21 (1437909123909812224)]} 0 8
[junit4:junit4] 2> 1904448 T3922 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[w!22 (1437909123922395136)]} 0 1
[junit4:junit4] 2> 1904459 T3972 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[w!22 (1437909123930783744)]} 0 1
[junit4:junit4] 2> 1904461 T3945 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[w!22 (1437909123930783744)]} 0 7
[junit4:junit4] 2> 1904464 T3957 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[w!22]} 0 12
[junit4:junit4] 2> 1904469 T3924 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[x!23 (1437909123944415232)]} 0 1
[junit4:junit4] 2> 1904480 T3991 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[x!23 (1437909123953852416)]} 0 1
[junit4:junit4] 2> 1904481 T3961 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[x!23 (1437909123953852416)]} 0 4
[junit4:junit4] 2> 1904483 T3941 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[x!23]} 0 10
[junit4:junit4] 2> 1904487 T3923 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[y!24 (1437909123963289600)]} 0 1
[junit4:junit4] 2> 1904493 T3992 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[y!24 (1437909123967483904)]} 0 1
[junit4:junit4] 2> 1904494 T3956 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[y!24 (1437909123967483904)]} 0 5
[junit4:junit4] 2> 1904498 T3925 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[z!25 (1437909123974823936)]} 0 1
[junit4:junit4] 2> 1904514 T3973 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[z!25 (1437909123989504000)]} 0 1
[junit4:junit4] 2> 1904515 T3940 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[z!25 (1437909123989504000)]} 0 4
[junit4:junit4] 2> 1904517 T3958 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[z!25]} 0 9
[junit4:junit4] 2> 1904520 T3920 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[a!26 (1437909123998941184)]} 0 0
[junit4:junit4] 2> 1904533 T3993 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[a!26 (1437909124006281216)]} 0 1
[junit4:junit4] 2> 1904534 T3960 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[a!26 (1437909124006281216)]} 0 7
[junit4:junit4] 2> 1904537 T3942 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[a!26]} 0 13
[junit4:junit4] 2> 1904541 T3921 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[b!27 (1437909124020961280)]} 0 1
[junit4:junit4] 2> 1904552 T3975 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[b!27 (1437909124029349888)]} 0 0
[junit4:junit4] 2> 1904554 T3943 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[b!27 (1437909124029349888)]} 0 5
[junit4:junit4] 2> 1904557 T3922 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[c!28 (1437909124037738496)]} 0 1
[junit4:junit4] 2> 1904566 T3974 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[c!28 (1437909124044029952)]} 0 1
[junit4:junit4] 2> 1904568 T3944 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[c!28 (1437909124044029952)]} 0 5
[junit4:junit4] 2> 1904573 T3959 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[c!28]} 0 13
[junit4:junit4] 2> 1904578 T3924 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[d!29 (1437909124059758592)]} 0 1
[junit4:junit4] 2> 1904585 T3994 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[d!29 (1437909124063952896)]} 0 1
[junit4:junit4] 2> 1904586 T3957 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[d!29 (1437909124063952896)]} 0 5
[junit4:junit4] 2> 1904590 T3923 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[e!30 (1437909124072341504)]} 0 0
[junit4:junit4] 2> 1904598 T3990 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[e!30 (1437909124076535808)]} 0 1
[junit4:junit4] 2> 1904599 T3961 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[e!30 (1437909124076535808)]} 0 5
[junit4:junit4] 2> 1904602 T3925 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[f!31 (1437909124084924416)]} 0 0
[junit4:junit4] 2> 1904610 T3989 C2921 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[f!31 (1437909124089118720)]} 0 1
[junit4:junit4] 2> 1904611 T3956 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[f!31 (1437909124089118720)]} 0 5
[junit4:junit4] 2> 1904615 T3920 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[g!32 (1437909124098555904)]} 0 0
[junit4:junit4] 2> 1904635 T3976 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[g!32 (1437909124110090240)]} 0 1
[junit4:junit4] 2> 1904637 T3945 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[g!32 (1437909124110090240)]} 0 12
[junit4:junit4] 2> 1904638 T3958 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[g!32]} 0 20
[junit4:junit4] 2> 1904645 T3921 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[h!33 (1437909124130013184)]} 0 0
[junit4:junit4] 2> 1904660 T3977 C2925 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[h!33 (1437909124138401792)]} 0 1
[junit4:junit4] 2> 1904665 T3941 C2924 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[h!33 (1437909124138401792)]} 0 12
[junit4:junit4] 2> 1904668 T3960 C2922 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[h!33]} 0 19
[junit4:junit4] 2> 1904677 T3922 C2923 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[i!34 (1437909124162519040)]} 0 2
[junit4:junit4] 2> ASYNC NEW_CORE C2926 name=collection1 org.apache.solr.core.SolrCore@10d64873 url=http://127.0.0.1:50275/js_y/collection1 node=127.0.0.1:50275_js_y C2926_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50275_js_y, base_url=http://127.0.0.1:50275/js_y}
[junit4:junit4] 2> 1904709 T3972 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[i!34 (1437909124187684864)]} 0 1
[junit4:junit4] 2> ASYNC NEW_CORE C2927 name=collection1 org.apache.solr.core.SolrCore@719c8968 url=http://127.0.0.1:50269/js_y/collection1 node=127.0.0.1:50269_js_y C2927_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50269_js_y, base_url=http://127.0.0.1:50269/js_y, leader=true}
[junit4:junit4] 2> 1904710 T3940 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[i!34 (1437909124187684864)]} 0 10
[junit4:junit4] 2> ASYNC NEW_CORE C2928 name=collection1 org.apache.solr.core.SolrCore@7a633564 url=http://127.0.0.1:50265/js_y/collection1 node=127.0.0.1:50265_js_y C2928_STATE=coll:control_collection core:collection1 props:{shard=shard1, state=active, core=collection1, collection=control_collection, node_name=127.0.0.1:50265_js_y, base_url=http://127.0.0.1:50265/js_y, leader=true}
[junit4:junit4] 2> 1904717 T3923 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[j!35 (1437909124205510656)]} 0 0
[junit4:junit4] 2> 1904725 T3973 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[j!35 (1437909124209704960)]} 0 1
[junit4:junit4] 2> 1904727 T3942 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[j!35 (1437909124209704960)]} 0 6
[junit4:junit4] 2> 1904731 T3925 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[k!36 (1437909124220190720)]} 0 0
[junit4:junit4] 2> 1904738 T3975 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[k!36 (1437909124224385024)]} 0 1
[junit4:junit4] 2> 1904739 T3943 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[k!36 (1437909124224385024)]} 0 5
[junit4:junit4] 2> 1904743 T3920 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[l!37 (1437909124231725056)]} 0 1
[junit4:junit4] 2> ASYNC NEW_CORE C2929 name=collection1 org.apache.solr.core.SolrCore@4b24cc95 url=http://127.0.0.1:50279/js_y/collection1 node=127.0.0.1:50279_js_y C2929_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50279_js_y, base_url=http://127.0.0.1:50279/js_y}
[junit4:junit4] 2> 1904750 T3991 C2929 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[l!37 (1437909124235919360)]} 0 0
[junit4:junit4] 2> ASYNC NEW_CORE C2930 name=collection1 org.apache.solr.core.SolrCore@6f1ce2c9 url=http://127.0.0.1:50272/js_y/collection1 node=127.0.0.1:50272_js_y C2930_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50272_js_y, base_url=http://127.0.0.1:50272/js_y, leader=true}
[junit4:junit4] 2> 1904752 T3959 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[l!37 (1437909124235919360)]} 0 6
[junit4:junit4] 2> 1904762 T3921 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[m!38 (1437909124252696576)]} 0 0
[junit4:junit4] 2> 1904769 T3992 C2929 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[m!38 (1437909124256890880)]} 0 0
[junit4:junit4] 2> 1904770 T3957 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[m!38 (1437909124256890880)]} 0 4
[junit4:junit4] 2> 1904776 T3922 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[n!39 (1437909124266328064)]} 0 1
[junit4:junit4] 2> 1904786 T3974 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[n!39 (1437909124274716672)]} 0 1
[junit4:junit4] 2> 1904787 T3944 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[n!39 (1437909124274716672)]} 0 5
[junit4:junit4] 2> 1904789 T3961 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[n!39]} 0 9
[junit4:junit4] 2> 1904793 T3924 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[o!40 (1437909124284153856)]} 0 1
[junit4:junit4] 2> 1904808 T3993 C2929 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[o!40 (1437909124291493888)]} 0 0
[junit4:junit4] 2> 1904810 T3956 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[o!40 (1437909124291493888)]} 0 12
[junit4:junit4] 2> 1904812 T3941 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[o!40]} 0 16
[junit4:junit4] 2> 1904815 T3923 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[p!41 (1437909124308271104)]} 0 0
[junit4:junit4] 2> 1904827 T3976 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[p!41 (1437909124316659712)]} 0 1
[junit4:junit4] 2> 1904828 T3940 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[p!41 (1437909124316659712)]} 0 5
[junit4:junit4] 2> 1904832 T3925 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[q!42 (1437909124326096896)]} 0 0
[junit4:junit4] 2> 1904843 T3972 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[q!42 (1437909124333436928)]} 0 1
[junit4:junit4] 2> 1904844 T3942 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[q!42 (1437909124333436928)]} 0 5
[junit4:junit4] 2> 1904845 T3958 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[q!42]} 0 9
[junit4:junit4] 2> 1904851 T3920 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[r!43 (1437909124344971264)]} 0 1
[junit4:junit4] 2> 1904862 T3994 C2929 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[r!43 (1437909124352311296)]} 0 1
[junit4:junit4] 2> 1904865 T3959 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[r!43 (1437909124352311296)]} 0 8
[junit4:junit4] 2> 1904868 T3943 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[r!43]} 0 14
[junit4:junit4] 2> 1904875 T3921 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[s!44 (1437909124371185664)]} 0 1
[junit4:junit4] 2> 1904883 T3973 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[s!44 (1437909124375379968)]} 0 0
[junit4:junit4] 2> 1904884 T3944 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[s!44 (1437909124375379968)]} 0 5
[junit4:junit4] 2> 1904888 T3922 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[t!45 (1437909124383768576)]} 0 1
[junit4:junit4] 2> 1904902 T3975 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[t!45 (1437909124391108608)]} 0 4
[junit4:junit4] 2> 1904906 T3945 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[t!45 (1437909124391108608)]} 0 12
[junit4:junit4] 2> 1904907 T3957 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[t!45]} 0 16
[junit4:junit4] 2> 1904911 T3924 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[u!46 (1437909124408934400)]} 0 0
[junit4:junit4] 2> 1904918 T3989 C2929 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[u!46 (1437909124413128704)]} 0 1
[junit4:junit4] 2> 1904919 T3961 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[u!46 (1437909124413128704)]} 0 4
[junit4:junit4] 2> 1904924 T3923 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[v!47 (1437909124422565888)]} 0 1
[junit4:junit4] 2> 1904934 T3974 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[v!47 (1437909124429905920)]} 0 1
[junit4:junit4] 2> 1904936 T3941 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[v!47 (1437909124429905920)]} 0 6
[junit4:junit4] 2> 1904937 T3956 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[v!47]} 0 9
[junit4:junit4] 2> 1904941 T3925 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[w!48 (1437909124440391680)]} 0 1
[junit4:junit4] 2> 1904948 T3977 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[w!48 (1437909124444585984)]} 0 1
[junit4:junit4] 2> 1904949 T3940 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[w!48 (1437909124444585984)]} 0 4
[junit4:junit4] 2> 1904952 T3920 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[x!49 (1437909124451926016)]} 0 0
[junit4:junit4] 2> 1904968 T3991 C2929 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[x!49 (1437909124460314624)]} 0 0
[junit4:junit4] 2> 1904970 T3960 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[x!49 (1437909124460314624)]} 0 12
[junit4:junit4] 2> 1904974 T3921 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[y!50 (1437909124474994688)]} 0 1
[junit4:junit4] 2> 1904988 T3992 C2929 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[y!50 (1437909124484431872)]} 0 1
[junit4:junit4] 2> 1904991 T3958 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[y!50 (1437909124484431872)]} 0 9
[junit4:junit4] 2> 1904998 T3922 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[z!51 (1437909124500160512)]} 0 1
[junit4:junit4] 2> 1905016 T3976 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[z!51 (1437909124512743424)]} 0 1
[junit4:junit4] 2> 1905018 T3942 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[z!51 (1437909124512743424)]} 0 8
[junit4:junit4] 2> 1905019 T3959 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[z!51]} 0 15
[junit4:junit4] 2> 1905023 T3924 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[a!52 (1437909124526374912)]} 0 0
[junit4:junit4] 2> 1905032 T3993 C2929 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[a!52 (1437909124532666368)]} 0 0
[junit4:junit4] 2> 1905033 T3957 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[a!52 (1437909124532666368)]} 0 4
[junit4:junit4] 2> 1905035 T3943 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[a!52]} 0 8
[junit4:junit4] 2> 1905043 T3923 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[b!53 (1437909124547346432)]} 0 1
[junit4:junit4] 2> 1905055 T3972 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[b!53 (1437909124556783616)]} 0 1
[junit4:junit4] 2> 1905056 T3944 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[b!53 (1437909124556783616)]} 0 5
[junit4:junit4] 2> 1905058 T3961 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[b!53]} 0 9
[junit4:junit4] 2> 1905062 T3925 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[c!54 (1437909124566220800)]} 0 1
[junit4:junit4] 2> 1905070 T3973 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[c!54 (1437909124571463680)]} 0 1
[junit4:junit4] 2> 1905071 T3945 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[c!54 (1437909124571463680)]} 0 5
[junit4:junit4] 2> 1905075 T3920 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[d!55 (1437909124579852288)]} 0 1
[junit4:junit4] 2> 1905090 T3990 C2929 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[d!55 (1437909124584046592)]} 0 4
[junit4:junit4] 2> 1905093 T3956 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[d!55 (1437909124584046592)]} 0 15
[junit4:junit4] 2> 1905097 T3921 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[e!56 (1437909124603969536)]} 0 1
[junit4:junit4] 2> 1905107 T3994 C2929 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[e!56 (1437909124610260992)]} 0 1
[junit4:junit4] 2> 1905109 T3960 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[e!56 (1437909124610260992)]} 0 6
[junit4:junit4] 2> 1905110 T3941 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[e!56]} 0 10
[junit4:junit4] 2> 1905114 T3924 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[f!57 (1437909124620746752)]} 0 1
[junit4:junit4] 2> 1905123 T3989 C2929 P50279 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[f!57 (1437909124627038208)]} 0 1
[junit4:junit4] 2> 1905124 T3958 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[f!57 (1437909124627038208)]} 0 5
[junit4:junit4] 2> 1905127 T3940 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[f!57]} 0 11
[junit4:junit4] 2> 1905132 T3923 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[g!58 (1437909124639621120)]} 0 1
[junit4:junit4] 2> 1905141 T3975 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[g!58 (1437909124645912576)]} 0 1
[junit4:junit4] 2> 1905142 T3942 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[g!58 (1437909124645912576)]} 0 6
[junit4:junit4] 2> 1905146 T3925 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[h!59 (1437909124655349760)]} 0 0
[junit4:junit4] 2> 1905155 T3974 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[h!59 (1437909124661641216)]} 0 1
[junit4:junit4] 2> 1905156 T3943 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[h!59 (1437909124661641216)]} 0 4
[junit4:junit4] 2> 1905157 T3959 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[h!59]} 0 8
[junit4:junit4] 2> 1905168 T3920 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[i!60 (1437909124677369856)]} 0 4
[junit4:junit4] 2> 1905178 T3977 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[i!60 (1437909124685758464)]} 0 1
[junit4:junit4] 2> 1905180 T3944 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[i!60 (1437909124685758464)]} 0 6
[junit4:junit4] 2> 1905181 T3957 C2930 P50272 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[i!60]} 0 9
[junit4:junit4] 2> 1905184 T3921 C2928 P50265 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={wt=javabin&version=2} {add=[j!61 (1437909124695195648)]} 0 0
[junit4:junit4] 2> 1905195 T3976 C2926 P50275 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50269/js_y/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[j!61 (1437909124702535680)]} 0 0
[junit4:junit4] 2> 1905197 T3945 C2927 P50269 oasup.LogUpdateProcessor.finish [collection1] webapp=/js_y path=/update params={distrib.from=http://127.0.0.1:50272/js_y/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[j!61 (1437909124702535680)]} 0 6
[junit4:junit4] 2> 1905199 T3961 C29
[...truncated too long message...]
Writer
[junit4:junit4] 2> 1940061 T3907 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
[junit4:junit4] 2> C2995_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=down, core=collection1, collection=collection1, node_name=127.0.0.1:50279_js_y, base_url=http://127.0.0.1:50279/js_y, leader=true}
[junit4:junit4] 2> 1940083 T3907 C2995 P50279 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@50f9409a lockFactory=org.apache.lucene.store.NativeFSLockFactory@45f45cb0),segFN=segments_4,generation=4}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@50f9409a lockFactory=org.apache.lucene.store.NativeFSLockFactory@45f45cb0),segFN=segments_5,generation=5}
[junit4:junit4] 2> 1940084 T3907 C2995 P50279 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 5
[junit4:junit4] 2> 1940111 T3907 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
[junit4:junit4] 2> 1940112 T3907 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
[junit4:junit4] 2> 1940113 T3907 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4;done=false>>]
[junit4:junit4] 2> 1940113 T3907 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4
[junit4:junit4] 2> 1940114 T3907 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4/index [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4/index;done=false>>]
[junit4:junit4] 2> 1940115 T3907 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ShardSplitTest-1371296967096/jetty4/index
[junit4:junit4] 2> 1940143 T4031 oasc.Overseer$ClusterStateUpdater.amILeader According to ZK I (id=89869318036520972-127.0.0.1:50279_js_y-n_0000000004) am no longer a leader.
[junit4:junit4] 2> 1940233 T3907 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/js_y,null}
[junit4:junit4] 2> 1940317 T3907 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
[junit4:junit4] 2> 1940323 T3907 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:50262 50262
[junit4:junit4] 2> 1940431 T3907 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 50265
[junit4:junit4] 2> !!!! WARNING: best effort to remove /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ShardSplitTest-1371296967096 FAILED !!!!!
[junit4:junit4] 2> 1940432 T3907 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
[junit4:junit4] 2> 1940433 T3907 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:50262 50262
[junit4:junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4:junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ShardSplitTest -Dtests.method=testDistribSearch -Dtests.seed=C15D2F8980D4126 -Dtests.slow=true -Dtests.locale=tr -Dtests.timezone=Indian/Mauritius -Dtests.file.encoding=UTF-8
[junit4:junit4] ERROR 69.7s | ShardSplitTest.testDistribSearch <<<
[junit4:junit4] > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:50269/js_y returned non ok status:500, message:Server Error
[junit4:junit4] > at __randomizedtesting.SeedInfo.seed([C15D2F8980D4126:8DF35CE0EF52211A]:0)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
[junit4:junit4] > at org.apache.solr.cloud.ShardSplitTest.splitShard(ShardSplitTest.java:228)
[junit4:junit4] > at org.apache.solr.cloud.ShardSplitTest.doTest(ShardSplitTest.java:150)
[junit4:junit4] > at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
[junit4:junit4] > at java.lang.Thread.run(Thread.java:722)
[junit4:junit4] 2> 1940457 T3907 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4:junit4] 2> 69752 T3906 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 2 leaked thread(s).
[junit4:junit4] 2> 1941525 T3998 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 1941525 T3998 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1941526 T3998 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> NOTE: test params are: codec=Lucene42: {id=PostingsFormat(name=SimpleText), _version_=PostingsFormat(name=Memory doPackFST= true), n_ti=Pulsing41(freqCutoff=1 minBlockSize=30 maxBlockSize=155)}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=tr, timezone=Indian/Mauritius
[junit4:junit4] 2> NOTE: Mac OS X 10.8.4 x86_64/Oracle Corporation 1.7.0_21 (64-bit)/cpus=2,threads=3,free=18333608,total=196567040
[junit4:junit4] 2> NOTE: All tests run in this JVM: [AlternateDirectoryTest, TestCopyFieldCollectionResource, TestSolrDeletionPolicy1, RequiredFieldsTest, TestTrie, TermsComponentTest, TestRangeQuery, TestDFRSimilarityFactory, TestSolrQueryParser, AnalysisAfterCoreReloadTest, PreAnalyzedUpdateProcessorTest, TestAddFieldRealTimeGet, TestStressVersions, StatsComponentTest, TestFoldingMultitermQuery, SuggesterTSTTest, QueryResultKeyTest, FieldMutatingUpdateProcessorTest, HighlighterConfigTest, TestLazyCores, DOMUtilTest, TestPhraseSuggestions, RAMDirectoryFactoryTest, XmlUpdateRequestHandlerTest, MinimalSchemaTest, DirectSolrSpellCheckerTest, TestPropInject, IndexSchemaTest, TestCodecSupport, TestUniqueKeyFieldResource, ZkNodePropsTest, TestSchemaSimilarityResource, TestSolrQueryParserResource, SliceStateTest, TestRemoteStreaming, DocumentAnalysisRequestHandlerTest, SuggesterFSTTest, OpenCloseCoreStressTest, TestValueSourceCache, ScriptEngineTest, BasicDistributedZkTest, CachingDirectoryFactoryTest, SyncSliceTest, TestWordDelimiterFilterFactory, TestPerFieldSimilarity, TestCloudManagedSchema, TestZkChroot, TestPseudoReturnFields, DistanceFunctionTest, AutoCommitTest, TestBadConfig, SolrCoreCheckLockOnStartupTest, TestPluginEnable, DistributedSpellCheckComponentTest, SliceStateUpdateTest, TestExtendedDismaxParser, TestRealTimeGet, CurrencyFieldXmlFileTest, OverseerCollectionProcessorTest, TestShardHandlerFactory, TestMaxScoreQueryParser, TestLFUCache, TestOmitPositions, TestSolrXml, SearchHandlerTest, ChaosMonkeyShardSplitTest, PreAnalyzedFieldTest, TestRandomDVFaceting, DebugComponentTest, LoggingHandlerTest, JsonLoaderTest, DistributedTermsComponentTest, SolrCmdDistributorTest, CoreContainerCoreInitFailuresTest, SolrIndexSplitterTest, TestCharFilters, TestLMDirichletSimilarityFactory, StatelessScriptUpdateProcessorFactoryTest, ReturnFieldsTest, TestPostingsSolrHighlighter, TestFastWriter, SolrTestCaseJ4Test, ResourceLoaderTest, TestDistributedSearch, TestFastOutputStream, BasicZkTest, TestFuzzyAnalyzedSuggestions, DisMaxRequestHandlerTest, NumericFieldsTest, TestSolrIndexConfig, TestPHPSerializedResponseWriter, QueryElevationComponentTest, SimplePostToolTest, BadIndexSchemaTest, CopyFieldTest, TestLMJelinekMercerSimilarityFactory, BasicDistributedZk2Test, SolrCoreTest, TestSchemaResource, FastVectorHighlighterTest, UpdateRequestProcessorFactoryTest, EchoParamsTest, TestReload, SampleTest, TestSweetSpotSimilarityFactory, TestJmxIntegration, TestUpdate, PeerSyncTest, TestSort, UniqFieldsUpdateProcessorFactoryTest, TimeZoneUtilsTest, SuggesterTest, DefaultValueUpdateProcessorTest, TestIndexSearcher, TestArbitraryIndexDir, TestAnalyzedSuggestions, TermVectorComponentTest, UpdateParamsTest, TestReplicationHandler, TestCSVResponseWriter, ChaosMonkeyNothingIsSafeTest, TestSolrDeletionPolicy2, TestJmxMonitoredMap, SpellingQueryConverterTest, TestDistributedGrouping, StandardRequestHandlerTest, TestMergePolicyConfig, ExternalFileFieldSortTest, CollectionsAPIDistributedZkTest, TestRandomFaceting, IndexSchemaRuntimeFieldTest, NotRequiredUniqueKeyTest, TestManagedSchemaFieldResource, TestFunctionQuery, ShardSplitTest]
[junit4:junit4] Completed in 71.08s, 1 test, 1 error <<< FAILURES!
[...truncated 473 lines...]
BUILD FAILED
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:386: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:366: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:39: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:181: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:437: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1243: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:887: There were test failures: 296 suites, 1243 tests, 1 error, 18 ignored (6 assumptions)
Total time: 106 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0 -XX:+UseCompressedOops -XX:+UseSerialGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure
[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.7.0) - Build # 562 -
Still Failing!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/562/
Java: 64bit/jdk1.7.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
2 tests failed.
REGRESSION: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([F3DD7ACAA9FDF51A]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([F3DD7ACAA9FDF51A]:0)
Build Log:
[...truncated 9664 lines...]
[junit4:junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
[junit4:junit4] 2> 1983495 T4320 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
[junit4:junit4] 2> 1983514 T4320 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4:junit4] 2> Creating dataDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./solrtest-ChaosMonkeySafeLeaderTest-1371254002311
[junit4:junit4] 2> 1983519 T4320 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4:junit4] 2> 1983523 T4321 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4:junit4] 2> 1983624 T4320 oasc.ZkTestServer.run start zk server on port:50470
[junit4:junit4] 2> 1983627 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1983635 T4327 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3d47ebbd name:ZooKeeperConnection Watcher:127.0.0.1:50470 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1983635 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1983636 T4320 oascc.SolrZkClient.makePath makePath: /solr
[junit4:junit4] 2> 1983647 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1983652 T4329 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a03292c name:ZooKeeperConnection Watcher:127.0.0.1:50470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1983652 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1983653 T4320 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4:junit4] 2> 1983667 T4320 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4:junit4] 2> 1983675 T4320 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4:junit4] 2> 1983683 T4320 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4:junit4] 2> 1983691 T4320 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 1983692 T4320 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 1983703 T4320 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4:junit4] 2> 1983704 T4320 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4:junit4] 2> 1983713 T4320 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4:junit4] 2> 1983714 T4320 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4:junit4] 2> 1983723 T4320 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4:junit4] 2> 1983725 T4320 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4:junit4] 2> 1983739 T4320 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4:junit4] 2> 1983741 T4320 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4:junit4] 2> 1983749 T4320 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 1983750 T4320 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 1983757 T4320 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 1983758 T4320 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 1983766 T4320 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 1983767 T4320 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 1983775 T4320 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4:junit4] 2> 1983776 T4320 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4:junit4] 2> 1984344 T4320 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1984356 T4320 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50473
[junit4:junit4] 2> 1984357 T4320 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1984357 T4320 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1984358 T4320 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371254002580
[junit4:junit4] 2> 1984358 T4320 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371254002580/solr.xml
[junit4:junit4] 2> 1984359 T4320 oasc.CoreContainer.<init> New CoreContainer 1291750359
[junit4:junit4] 2> 1984360 T4320 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371254002580/'
[junit4:junit4] 2> 1984360 T4320 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371254002580/'
[junit4:junit4] 2> 1984560 T4320 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1984561 T4320 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1984561 T4320 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1984562 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1984563 T4320 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1984563 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1984564 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1984565 T4320 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1984565 T4320 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1984566 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1984599 T4320 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1984600 T4320 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50470/solr
[junit4:junit4] 2> 1984601 T4320 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1984603 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1984608 T4340 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@bb1e18f name:ZooKeeperConnection Watcher:127.0.0.1:50470 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1984609 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1984614 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1984639 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1984646 T4342 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5550fdc4 name:ZooKeeperConnection Watcher:127.0.0.1:50470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1984646 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1984650 T4320 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4:junit4] 2> 1984665 T4320 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4:junit4] 2> 1984676 T4320 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4:junit4] 2> 1984681 T4320 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50473_
[junit4:junit4] 2> 1984683 T4320 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50473_
[junit4:junit4] 2> 1984692 T4320 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4:junit4] 2> 1984706 T4320 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4:junit4] 2> 1984717 T4320 oasc.Overseer.start Overseer (id=89866502296567811-127.0.0.1:50473_-n_0000000000) starting
[junit4:junit4] 2> 1984726 T4320 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4:junit4] 2> 1984743 T4344 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4:junit4] 2> 1984745 T4343 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4:junit4] 2> 1984748 T4320 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4:junit4] 2> 1984756 T4320 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4:junit4] 2> 1984762 T4320 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1984782 T4345 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1984783 T4345 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1986279 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1986285 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50473_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50473"}
[junit4:junit4] 2> 1986285 T4343 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
[junit4:junit4] 2> 1986285 T4343 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1986298 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1986795 T4345 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371254002580/collection1
[junit4:junit4] 2> 1986795 T4345 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4:junit4] 2> 1986797 T4345 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1986798 T4345 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4:junit4] 2> 1986801 T4345 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371254002580/collection1/'
[junit4:junit4] 2> 1986803 T4345 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371254002580/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1986804 T4345 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371254002580/collection1/lib/README' to classloader
[junit4:junit4] 2> 1986911 T4345 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1987017 T4345 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1987021 T4345 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1987029 T4345 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1988045 T4345 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1988069 T4345 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1988073 T4345 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1988106 T4345 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1988114 T4345 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1988121 T4345 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1988130 T4345 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1988130 T4345 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1988130 T4345 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1988135 T4345 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1988136 T4345 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1988136 T4345 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1988137 T4345 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371254002580/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/control/data/
[junit4:junit4] 2> 1988137 T4345 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@269f046
[junit4:junit4] 2> 1988138 T4345 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1988139 T4345 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/control/data
[junit4:junit4] 2> 1988141 T4345 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/control/data/index/
[junit4:junit4] 2> 1988142 T4345 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/control/data/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1988143 T4345 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/control/data/index
[junit4:junit4] 2> 1988153 T4345 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/control/data/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1988153 T4345 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1988159 T4345 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1988159 T4345 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1988160 T4345 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1988161 T4345 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1988162 T4345 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1988162 T4345 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1988163 T4345 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1988164 T4345 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1988165 T4345 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1988194 T4345 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1988207 T4345 oass.SolrIndexSearcher.<init> Opening Searcher@daf2605 main
[junit4:junit4] 2> 1988209 T4345 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1988210 T4345 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1988216 T4346 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@daf2605 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1988218 T4345 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1988219 T4345 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50473 collection:control_collection shard:shard1
[junit4:junit4] 2> 1988231 T4345 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4:junit4] 2> 1988575 T4345 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 1988589 T4345 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 1988589 T4345 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 1988589 T4345 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50473/collection1/
[junit4:junit4] 2> 1988590 T4345 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 1988590 T4345 oasc.SyncStrategy.syncToMe http://127.0.0.1:50473/collection1/ has no replicas
[junit4:junit4] 2> 1988590 T4345 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50473/collection1/
[junit4:junit4] 2> 1988591 T4345 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4:junit4] 2> 1989325 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1989348 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1989380 T4345 oasc.ZkController.register We are http://127.0.0.1:50473/collection1/ and leader is http://127.0.0.1:50473/collection1/
[junit4:junit4] 2> 1989380 T4345 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50473
[junit4:junit4] 2> 1989381 T4345 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 1989382 T4345 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1989384 T4345 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1989391 T4345 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1989394 T4320 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1989395 T4320 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1989395 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1989425 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:
[junit4:junit4] 2> 1989428 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1989435 T4349 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@8baf3e4 name:ZooKeeperConnection Watcher:127.0.0.1:50470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1989435 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1989438 T4320 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1989445 T4320 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
[junit4:junit4] 2> 1990000 T4320 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1990007 T4320 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50477
[junit4:junit4] 2> 1990010 T4320 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1990011 T4320 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1990011 T4320 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371254008241
[junit4:junit4] 2> 1990012 T4320 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371254008241/solr.xml
[junit4:junit4] 2> 1990013 T4320 oasc.CoreContainer.<init> New CoreContainer 2124776185
[junit4:junit4] 2> 1990014 T4320 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371254008241/'
[junit4:junit4] 2> 1990014 T4320 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371254008241/'
[junit4:junit4] 2> 1990214 T4320 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1990216 T4320 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1990216 T4320 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1990217 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1990217 T4320 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1990218 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1990219 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1990219 T4320 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1990220 T4320 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1990220 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1990267 T4320 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1990268 T4320 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50470/solr
[junit4:junit4] 2> 1990268 T4320 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1990270 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1990281 T4360 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e7aaa11 name:ZooKeeperConnection Watcher:127.0.0.1:50470 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1990282 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1990288 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1990317 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1990322 T4362 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@643b0ec3 name:ZooKeeperConnection Watcher:127.0.0.1:50470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1990322 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1990333 T4320 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1990867 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1990869 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50473_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50473"}
[junit4:junit4] 2> 1990880 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1990880 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1990881 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1991342 T4320 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50477_
[junit4:junit4] 2> 1991345 T4320 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50477_
[junit4:junit4] 2> 1991351 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1991352 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1991355 T4362 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 1991356 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1991363 T4342 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 1991363 T4349 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 1991372 T4363 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1991373 T4363 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1992397 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1992399 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50477_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50477"}
[junit4:junit4] 2> 1992399 T4343 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 3
[junit4:junit4] 2> 1992399 T4343 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1992411 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1992411 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1992412 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1993377 T4363 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371254008241/collection1
[junit4:junit4] 2> 1993377 T4363 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1993379 T4363 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1993379 T4363 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1993383 T4363 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371254008241/collection1/'
[junit4:junit4] 2> 1993385 T4363 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371254008241/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1993385 T4363 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371254008241/collection1/lib/README' to classloader
[junit4:junit4] 2> 1993470 T4363 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1993567 T4363 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1993571 T4363 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1993578 T4363 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1994564 T4363 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1994577 T4363 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1994582 T4363 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1994613 T4363 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1994627 T4363 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1994635 T4363 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1994638 T4363 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1994641 T4363 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1994641 T4363 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1994645 T4363 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1994645 T4363 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1994645 T4363 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1994646 T4363 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371254008241/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty1/
[junit4:junit4] 2> 1994646 T4363 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@269f046
[junit4:junit4] 2> 1994647 T4363 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1994648 T4363 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty1
[junit4:junit4] 2> 1994649 T4363 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty1/index/
[junit4:junit4] 2> 1994650 T4363 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty1/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1994651 T4363 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty1/index
[junit4:junit4] 2> 1994658 T4363 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty1/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1994658 T4363 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1994663 T4363 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1994664 T4363 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1994665 T4363 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1994666 T4363 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1994667 T4363 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1994667 T4363 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1994668 T4363 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1994669 T4363 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1994669 T4363 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1994692 T4363 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1994706 T4363 oass.SolrIndexSearcher.<init> Opening Searcher@47e8fbbb main
[junit4:junit4] 2> 1994707 T4363 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1994708 T4363 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1994713 T4364 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@47e8fbbb main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1994717 T4363 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1994717 T4363 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50477 collection:collection1 shard:shard1
[junit4:junit4] 2> 1994719 T4363 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4:junit4] 2> 1994737 T4363 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 1994744 T4363 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 1994745 T4363 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 1994745 T4363 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50477/collection1/
[junit4:junit4] 2> 1994745 T4363 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 1994745 T4363 oasc.SyncStrategy.syncToMe http://127.0.0.1:50477/collection1/ has no replicas
[junit4:junit4] 2> 1994746 T4363 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50477/collection1/
[junit4:junit4] 2> 1994746 T4363 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4:junit4] 2> 1995460 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1995479 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1995480 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1995485 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1995547 T4363 oasc.ZkController.register We are http://127.0.0.1:50477/collection1/ and leader is http://127.0.0.1:50477/collection1/
[junit4:junit4] 2> 1995551 T4363 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50477
[junit4:junit4] 2> 1995552 T4363 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 1995552 T4363 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1995552 T4363 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1995558 T4363 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1995560 T4320 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1995561 T4320 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1995562 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1996102 T4320 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1996160 T4320 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50480
[junit4:junit4] 2> 1996161 T4320 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1996162 T4320 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1996162 T4320 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371254014382
[junit4:junit4] 2> 1996163 T4320 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371254014382/solr.xml
[junit4:junit4] 2> 1996163 T4320 oasc.CoreContainer.<init> New CoreContainer 1002335507
[junit4:junit4] 2> 1996164 T4320 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371254014382/'
[junit4:junit4] 2> 1996164 T4320 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371254014382/'
[junit4:junit4] 2> 1996359 T4320 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1996362 T4320 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1996363 T4320 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1996363 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1996364 T4320 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1996364 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1996365 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1996365 T4320 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1996366 T4320 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1996366 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1996400 T4320 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1996401 T4320 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50470/solr
[junit4:junit4] 2> 1996401 T4320 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1996405 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1996409 T4376 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@37a507ee name:ZooKeeperConnection Watcher:127.0.0.1:50470 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1996410 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1996414 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1996436 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1996442 T4378 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@587d050c name:ZooKeeperConnection Watcher:127.0.0.1:50470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1996442 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1996452 T4320 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1996998 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1996999 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50477_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50477"}
[junit4:junit4] 2> 1997011 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1997011 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1997011 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1997012 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1997463 T4320 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50480_
[junit4:junit4] 2> 1997470 T4320 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50480_
[junit4:junit4] 2> 1997482 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1997485 T4378 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1997485 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1997491 T4342 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1997494 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1997508 T4349 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1997508 T4362 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1997508 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1997513 T4379 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1997518 T4379 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1998527 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1998529 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50480_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50480"}
[junit4:junit4] 2> 1998529 T4343 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4] 2> 1998529 T4343 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 1998540 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1998541 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1998541 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1998541 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1999540 T4379 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371254014382/collection1
[junit4:junit4] 2> 1999541 T4379 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1999543 T4379 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1999543 T4379 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1999547 T4379 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371254014382/collection1/'
[junit4:junit4] 2> 1999548 T4379 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371254014382/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1999549 T4379 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371254014382/collection1/lib/README' to classloader
[junit4:junit4] 2> 1999661 T4379 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1999776 T4379 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1999781 T4379 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1999790 T4379 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 2000800 T4379 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 2000821 T4379 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 2000826 T4379 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 2000861 T4379 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2000870 T4379 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2000879 T4379 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 2000883 T4379 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 2000884 T4379 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 2000885 T4379 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 2000888 T4379 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 2000888 T4379 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 2000889 T4379 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 2000891 T4379 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371254014382/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty2/
[junit4:junit4] 2> 2000893 T4379 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@269f046
[junit4:junit4] 2> 2000898 T4379 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 2000901 T4379 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty2
[junit4:junit4] 2> 2000903 T4379 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty2/index/
[junit4:junit4] 2> 2000904 T4379 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty2/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 2000906 T4379 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty2/index
[junit4:junit4] 2> 2000916 T4379 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty2/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 2000917 T4379 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2000924 T4379 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 2000925 T4379 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 2000926 T4379 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 2000927 T4379 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 2000928 T4379 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 2000928 T4379 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 2000928 T4379 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 2000929 T4379 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 2000930 T4379 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 2000952 T4379 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 2000969 T4379 oass.SolrIndexSearcher.<init> Opening Searcher@4f41630a main
[junit4:junit4] 2> 2000971 T4379 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 2000971 T4379 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 2000977 T4380 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4f41630a main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 2000981 T4379 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 2000981 T4379 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50480 collection:collection1 shard:shard2
[junit4:junit4] 2> 2000983 T4379 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4:junit4] 2> 2001000 T4379 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 2001007 T4379 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 2001007 T4379 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 2001008 T4379 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50480/collection1/
[junit4:junit4] 2> 2001008 T4379 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 2001008 T4379 oasc.SyncStrategy.syncToMe http://127.0.0.1:50480/collection1/ has no replicas
[junit4:junit4] 2> 2001009 T4379 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50480/collection1/
[junit4:junit4] 2> 2001009 T4379 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4:junit4] 2> 2001579 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2001610 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2001611 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2001611 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2001612 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2001637 T4379 oasc.ZkController.register We are http://127.0.0.1:50480/collection1/ and leader is http://127.0.0.1:50480/collection1/
[junit4:junit4] 2> 2001638 T4379 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50480
[junit4:junit4] 2> 2001638 T4379 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 2001638 T4379 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 2001639 T4379 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2001643 T4379 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2001647 T4320 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 2001647 T4320 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 2001648 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2002496 T4320 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 2002503 T4320 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50483
[junit4:junit4] 2> 2002506 T4320 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 2002507 T4320 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 2002507 T4320 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371254020482
[junit4:junit4] 2> 2002508 T4320 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371254020482/solr.xml
[junit4:junit4] 2> 2002509 T4320 oasc.CoreContainer.<init> New CoreContainer 1060657269
[junit4:junit4] 2> 2002509 T4320 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371254020482/'
[junit4:junit4] 2> 2002510 T4320 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371254020482/'
[junit4:junit4] 2> 2002723 T4320 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 2002724 T4320 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 2002725 T4320 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 2002725 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 2002726 T4320 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 2002726 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 2002727 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 2002728 T4320 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 2002729 T4320 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 2002729 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 2002761 T4320 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 2002762 T4320 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50470/solr
[junit4:junit4] 2> 2002763 T4320 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 2002765 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2002771 T4392 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a3eea7f name:ZooKeeperConnection Watcher:127.0.0.1:50470 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2002772 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2002777 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 2002802 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2002805 T4394 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@70915b46 name:ZooKeeperConnection Watcher:127.0.0.1:50470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2002806 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2002823 T4320 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 2003123 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2003125 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50480_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50480"}
[junit4:junit4] 2> 2003135 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2003135 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2003136 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2003136 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2003136 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2003835 T4320 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50483_
[junit4:junit4] 2> 2003838 T4320 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50483_
[junit4:junit4] 2> 2003846 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2003848 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2003850 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2003855 T4378 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 2003855 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2003855 T4362 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 2003856 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2003866 T4342 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 2003866 T4349 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 2003867 T4394 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 2003879 T4395 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 2003879 T4395 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2004657 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2004659 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50483_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50483"}
[junit4:junit4] 2> 2004660 T4343 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4] 2> 2004660 T4343 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard3
[junit4:junit4] 2> 2004672 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2004672 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2004673 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2004672 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2004673 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2004884 T4395 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371254020482/collection1
[junit4:junit4] 2> 2004885 T4395 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 2004886 T4395 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 2004887 T4395 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 2004890 T4395 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371254020482/collection1/'
[junit4:junit4] 2> 2004891 T4395 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371254020482/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 2004891 T4395 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371254020482/collection1/lib/README' to classloader
[junit4:junit4] 2> 2004976 T4395 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 2005082 T4395 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 2005090 T4395 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 2005097 T4395 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 2006124 T4395 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 2006139 T4395 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 2006144 T4395 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 2006179 T4395 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2006190 T4395 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2006207 T4395 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 2006211 T4395 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 2006213 T4395 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 2006214 T4395 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 2006216 T4395 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 2006217 T4395 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 2006217 T4395 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 2006218 T4395 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371254020482/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty3/
[junit4:junit4] 2> 2006218 T4395 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@269f046
[junit4:junit4] 2> 2006219 T4395 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 2006220 T4395 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty3
[junit4:junit4] 2> 2006221 T4395 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty3/index/
[junit4:junit4] 2> 2006222 T4395 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty3/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 2006224 T4395 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty3/index
[junit4:junit4] 2> 2006231 T4395 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty3/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 2006232 T4395 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2006237 T4395 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 2006238 T4395 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 2006239 T4395 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 2006240 T4395 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 2006241 T4395 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 2006241 T4395 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 2006242 T4395 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 2006269 T4395 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 2006270 T4395 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 2006296 T4395 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 2006309 T4395 oass.SolrIndexSearcher.<init> Opening Searcher@567f415d main
[junit4:junit4] 2> 2006311 T4395 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 2006311 T4395 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 2006317 T4396 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@567f415d main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 2006326 T4395 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 2006327 T4395 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50483 collection:collection1 shard:shard3
[junit4:junit4] 2> 2006329 T4395 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
[junit4:junit4] 2> 2006347 T4395 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 2006355 T4395 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 2006356 T4395 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 2006356 T4395 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50483/collection1/
[junit4:junit4] 2> 2006357 T4395 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 2006357 T4395 oasc.SyncStrategy.syncToMe http://127.0.0.1:50483/collection1/ has no replicas
[junit4:junit4] 2> 2006357 T4395 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50483/collection1/
[junit4:junit4] 2> 2006358 T4395 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
[junit4:junit4] 2> 2007704 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2007734 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2007734 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2007735 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2007735 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2007735 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2007772 T4395 oasc.ZkController.register We are http://127.0.0.1:50483/collection1/ and leader is http://127.0.0.1:50483/collection1/
[junit4:junit4] 2> 2007773 T4395 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50483
[junit4:junit4] 2> 2007773 T4395 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 2007774 T4395 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 2007774 T4395 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2007782 T4395 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2007789 T4320 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 2007790 T4320 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 2007791 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2008387 T4320 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 2008396 T4320 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50486
[junit4:junit4] 2> 2008402 T4320 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 2008407 T4320 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 2008408 T4320 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371254026612
[junit4:junit4] 2> 2008409 T4320 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371254026612/solr.xml
[junit4:junit4] 2> 2008410 T4320 oasc.CoreContainer.<init> New CoreContainer 466781985
[junit4:junit4] 2> 2008410 T4320 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371254026612/'
[junit4:junit4] 2> 2008411 T4320 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371254026612/'
[junit4:junit4] 2> 2008629 T4320 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 2008630 T4320 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 2008631 T4320 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 2008631 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 2008632 T4320 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 2008632 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 2008633 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 2008633 T4320 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 2008634 T4320 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 2008635 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 2008666 T4320 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 2008667 T4320 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50470/solr
[junit4:junit4] 2> 2008668 T4320 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 2008670 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2008674 T4408 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@76769df6 name:ZooKeeperConnection Watcher:127.0.0.1:50470 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2008676 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2008680 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 2008703 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2008708 T4410 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1aca2c25 name:ZooKeeperConnection Watcher:127.0.0.1:50470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2008708 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2008723 T4320 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 2009257 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2009259 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard3",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50483_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50483"}
[junit4:junit4] 2> 2009273 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2009275 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2009276 T4410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2009278 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2009280 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2009281 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2009735 T4320 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50486_
[junit4:junit4] 2> 2009743 T4320 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50486_
[junit4:junit4] 2> 2009755 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2009760 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2009760 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2009763 T4410 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 2009763 T4410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2009763 T4362 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 2009764 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2009763 T4378 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 2009765 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2009772 T4394 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 2009773 T4349 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 2009773 T4342 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 2009789 T4411 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 2009789 T4411 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2010799 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2010801 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50486_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50486"}
[junit4:junit4] 2> 2010803 T4343 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4] 2> 2010803 T4343 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 2010816 T4410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2010817 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2010817 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2010818 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2010832 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2010835 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2011794 T4411 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371254026612/collection1
[junit4:junit4] 2> 2011795 T4411 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 2011797 T4411 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 2011798 T4411 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 2011807 T4411 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371254026612/collection1/'
[junit4:junit4] 2> 2011811 T4411 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371254026612/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 2011812 T4411 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371254026612/collection1/lib/README' to classloader
[junit4:junit4] 2> 2011921 T4411 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 2012058 T4411 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 2012061 T4411 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 2012075 T4411 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 2013119 T4411 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 2013135 T4411 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 2013145 T4411 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 2013185 T4411 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2013195 T4411 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2013205 T4411 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 2013209 T4411 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 2013210 T4411 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 2013210 T4411 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 2013213 T4411 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 2013213 T4411 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 2013213 T4411 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 2013214 T4411 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371254026612/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty4/
[junit4:junit4] 2> 2013214 T4411 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@269f046
[junit4:junit4] 2> 2013215 T4411 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 2013217 T4411 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty4
[junit4:junit4] 2> 2013218 T4411 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty4/index/
[junit4:junit4] 2> 2013221 T4411 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty4/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 2013225 T4411 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty4/index
[junit4:junit4] 2> 2013233 T4411 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty4/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 2013233 T4411 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2013240 T4411 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 2013241 T4411 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 2013242 T4411 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 2013243 T4411 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 2013244 T4411 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 2013244 T4411 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 2013245 T4411 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 2013246 T4411 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 2013246 T4411 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 2013265 T4411 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 2013284 T4411 oass.SolrIndexSearcher.<init> Opening Searcher@574ed0d7 main
[junit4:junit4] 2> 2013286 T4411 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 2013286 T4411 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 2013292 T4412 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@574ed0d7 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 2013295 T4411 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 2013295 T4411 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50486 collection:collection1 shard:shard1
[junit4:junit4] 2> 2013304 T4411 oasc.ZkController.register We are http://127.0.0.1:50486/collection1/ and leader is http://127.0.0.1:50477/collection1/
[junit4:junit4] 2> 2013304 T4411 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50486
[junit4:junit4] 2> 2013305 T4411 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 2013305 T4411 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C1973 name=collection1 org.apache.solr.core.SolrCore@50a6b10d url=http://127.0.0.1:50486/collection1 node=127.0.0.1:50486_ C1973_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:50486_, base_url=http://127.0.0.1:50486}
[junit4:junit4] 2> 2013306 T4413 C1973 P50486 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 2013307 T4411 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2013308 T4413 C1973 P50486 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 2013309 T4413 C1973 P50486 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 2013309 T4413 C1973 P50486 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2013312 T4320 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 2013313 T4320 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 2013313 T4413 C1973 P50486 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2013314 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2013353 T4356 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 2013851 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2013852 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50486_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50486"}
[junit4:junit4] 2> 2013866 T4410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2013866 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2013867 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2013867 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2013868 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2013869 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2013954 T4320 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 2013963 T4320 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50490
[junit4:junit4] 2> 2013965 T4320 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 2013965 T4320 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 2013966 T4320 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371254032144
[junit4:junit4] 2> 2013966 T4320 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371254032144/solr.xml
[junit4:junit4] 2> 2013967 T4320 oasc.CoreContainer.<init> New CoreContainer 520228104
[junit4:junit4] 2> 2013968 T4320 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371254032144/'
[junit4:junit4] 2> 2013968 T4320 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371254032144/'
[junit4:junit4] 2> 2014171 T4320 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 2014172 T4320 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 2014173 T4320 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 2014173 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 2014174 T4320 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 2014175 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 2014180 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 2014182 T4320 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 2014183 T4320 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 2014183 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 2014211 T4320 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 2014212 T4320 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50470/solr
[junit4:junit4] 2> 2014213 T4320 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 2014216 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2014224 T4425 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45b77355 name:ZooKeeperConnection Watcher:127.0.0.1:50470 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2014224 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2014230 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 2014260 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2014265 T4427 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a717b43 name:ZooKeeperConnection Watcher:127.0.0.1:50470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2014265 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2014279 T4320 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 2014355 T4356 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 2014356 T4356 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=5&state=recovering&nodeName=127.0.0.1:50486_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1003
[junit4:junit4] 2> 2015289 T4320 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50490_
[junit4:junit4] 2> 2015291 T4320 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50490_
[junit4:junit4] 2> 2015307 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2015318 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2015318 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2015322 T4362 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 2015323 T4427 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 2015323 T4378 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 2015324 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2015322 T4410 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 2015328 T4410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2015323 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2015331 T4342 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 2015336 T4394 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 2015337 T4349 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 2015351 T4428 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 2015352 T4428 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2015395 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2015397 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"6",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50490_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50490"}
[junit4:junit4] 2> 2015398 T4343 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4] 2> 2015398 T4343 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 2015417 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2015418 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2015418 T4427 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2015419 T4410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2015419 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2015420 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2015420 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> C1973_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:50486_, base_url=http://127.0.0.1:50486}
[junit4:junit4] 2> 2016359 T4413 C1973 P50486 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:50477/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 2016359 T4413 C1973 P50486 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:50486 START replicas=[http://127.0.0.1:50477/collection1/] nUpdates=100
[junit4:junit4] 2> 2016360 T4413 C1973 P50486 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 2016363 T4413 C1973 P50486 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 2016363 T4413 C1973 P50486 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 2016363 T4413 C1973 P50486 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 2016363 T4413 C1973 P50486 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 2016364 T4413 C1973 P50486 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:50477/collection1/. core=collection1
[junit4:junit4] 2> 2016364 T4413 C1973 P50486 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2016362 T4428 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371254032144/collection1
[junit4:junit4] 2> 2016374 T4428 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 2016387 T4428 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 2016388 T4428 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> ASYNC NEW_CORE C1974 name=collection1 org.apache.solr.core.SolrCore@42dc50f3 url=http://127.0.0.1:50477/collection1 node=127.0.0.1:50477_ C1974_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50477_, base_url=http://127.0.0.1:50477, leader=true}
[junit4:junit4] 2> 2016396 T4356 C1974 P50477 oasc.SolrCore.execute [collection1] webapp= path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=1
[junit4:junit4] 2> 2016400 T4428 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371254032144/collection1/'
[junit4:junit4] 2> 2016403 T4428 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371254032144/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 2016404 T4428 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371254032144/collection1/lib/README' to classloader
[junit4:junit4] 2> 2016405 T4353 C1974 P50477 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 2016414 T4353 C1974 P50477 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty1/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 2016416 T4353 C1974 P50477 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2016424 T4353 C1974 P50477 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty1/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty1/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 2016425 T4353 C1974 P50477 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 2016427 T4353 C1974 P50477 oass.SolrIndexSearcher.<init> Opening Searcher@4ef2b123 realtime
[junit4:junit4] 2> 2016428 T4353 C1974 P50477 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 2016429 T4353 C1974 P50477 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 24
[junit4:junit4] 2> 2016431 T4413 C1973 P50486 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 2016431 T4413 C1973 P50486 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 2016448 T4357 C1974 P50477 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 2016449 T4357 C1974 P50477 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=2
[junit4:junit4] 2> 2016451 T4413 C1973 P50486 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 2016451 T4413 C1973 P50486 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 2016451 T4413 C1973 P50486 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 2016465 T4355 C1974 P50477 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=3
[junit4:junit4] 2> 2016467 T4413 C1973 P50486 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 2016469 T4413 C1973 P50486 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty4/index.20130615015355262
[junit4:junit4] 2> 2016470 T4413 C1973 P50486 oash.SnapPuller.fetchLatestIndex Starting download to org.apache.lucene.store.NIOFSDirectory@/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty4/index.20130615015355262 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4900569c fullCopy=false
[junit4:junit4] 2> 2016476 T4358 C1974 P50477 oasc.SolrCore.execute [collection1] webapp= path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=1
[junit4:junit4] 2> 2016483 T4413 C1973 P50486 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 2016497 T4413 C1973 P50486 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 2016505 T4413 C1973 P50486 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 2016512 T4413 C1973 P50486 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty4/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty4/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 2016513 T4413 C1973 P50486 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 2016513 T4413 C1973 P50486 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 2016514 T4413 C1973 P50486 oass.SolrIndexSearcher.<init> Opening Searcher@274ac0ab main
[junit4:junit4] 2> 2016516 T4412 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@274ac0ab main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 2016517 T4413 C1973 P50486 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty4/index.20130615015355262 [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty4/index.20130615015355262;done=true>>]
[junit4:junit4] 2> 2016517 T4413 C1973 P50486 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty4/index.20130615015355262
[junit4:junit4] 2> 2016517 T4413 C1973 P50486 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty4/index.20130615015355262
[junit4:junit4] 2> 2016519 T4413 C1973 P50486 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 2016520 T4413 C1973 P50486 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 2016522 T4413 C1973 P50486 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 2016524 T4413 C1973 P50486 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2016530 T4413 C1973 P50486 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 2016616 T4428 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 2016705 T4428 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 2016713 T4428 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 2016727 T4428 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 2016941 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2016943 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50486_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50486"}
[junit4:junit4] 2> 2016969 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2016969 T4410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2016969 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2016970 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2016969 T4427 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2016970 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2016970 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2017752 T4428 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 2017767 T4428 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 2017773 T4428 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 2017831 T4428 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2017840 T4428 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2017849 T4428 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 2017859 T4428 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 2017862 T4428 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 2017862 T4428 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 2017871 T4428 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 2017872 T4428 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 2017873 T4428 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 2017874 T4428 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371254032144/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty5/
[junit4:junit4] 2> 2017874 T4428 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@269f046
[junit4:junit4] 2> 2017875 T4428 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 2017878 T4428 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty5
[junit4:junit4] 2> 2017879 T4428 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty5/index/
[junit4:junit4] 2> 2017881 T4428 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty5/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 2017882 T4428 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty5/index
[junit4:junit4] 2> 2017892 T4428 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty5/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 2017893 T4428 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2017900 T4428 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 2017902 T4428 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 2017903 T4428 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 2017905 T4428 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 2017906 T4428 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 2017906 T4428 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 2017912 T4428 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 2017913 T4428 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 2017915 T4428 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 2017956 T4428 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 2017991 T4428 oass.SolrIndexSearcher.<init> Opening Searcher@69282839 main
[junit4:junit4] 2> 2017993 T4428 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 2017993 T4428 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 2017999 T4431 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@69282839 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 2018003 T4428 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 2018004 T4428 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50490 collection:collection1 shard:shard2
[junit4:junit4] 2> 2018012 T4428 oasc.ZkController.register We are http://127.0.0.1:50490/collection1/ and leader is http://127.0.0.1:50480/collection1/
[junit4:junit4] 2> 2018012 T4428 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50490
[junit4:junit4] 2> 2018012 T4428 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 2018013 T4428 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C1975 name=collection1 org.apache.solr.core.SolrCore@13df4bf7 url=http://127.0.0.1:50490/collection1 node=127.0.0.1:50490_ C1975_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:50490_, base_url=http://127.0.0.1:50490}
[junit4:junit4] 2> 2018014 T4432 C1975 P50490 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 2018024 T4432 C1975 P50490 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 2018025 T4432 C1975 P50490 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 2018026 T4432 C1975 P50490 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2018036 T4428 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2018038 T4432 C1975 P50490 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2018061 T4320 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 2018066 T4320 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 2018067 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2018088 T4369 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 6, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 2018494 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2018498 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"6",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50490_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50490"}
[junit4:junit4] 2> 2018513 T4427 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2018514 T4410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2018514 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2018515 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2018515 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2018516 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2018517 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2018855 T4320 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 2018867 T4320 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50497
[junit4:junit4] 2> 2018869 T4320 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 2018870 T4320 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 2018871 T4320 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371254036894
[junit4:junit4] 2> 2018872 T4320 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371254036894/solr.xml
[junit4:junit4] 2> 2018873 T4320 oasc.CoreContainer.<init> New CoreContainer 1986712692
[junit4:junit4] 2> 2018874 T4320 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371254036894/'
[junit4:junit4] 2> 2018875 T4320 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371254036894/'
[junit4:junit4] 2> 2019092 T4369 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 6, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 2019093 T4369 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=6&state=recovering&nodeName=127.0.0.1:50490_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1005
[junit4:junit4] 2> 2019274 T4320 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 2019275 T4320 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 2019276 T4320 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 2019276 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 2019280 T4320 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 2019285 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 2019287 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 2019288 T4320 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 2019289 T4320 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 2019290 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 2019319 T4320 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 2019321 T4320 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50470/solr
[junit4:junit4] 2> 2019321 T4320 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 2019326 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2019333 T4444 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@763717a4 name:ZooKeeperConnection Watcher:127.0.0.1:50470 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2019335 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2019342 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 2019405 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2019416 T4446 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@669a0a6c name:ZooKeeperConnection Watcher:127.0.0.1:50470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2019417 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2019438 T4320 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 2020451 T4320 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50497_
[junit4:junit4] 2> 2020455 T4320 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50497_
[junit4:junit4] 2> 2020465 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2020466 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2020473 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 2020478 T4446 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 2020478 T4362 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 2020479 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2020480 T4427 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 2020480 T4427 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2020481 T4410 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 2020482 T4410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2020499 T4349 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 2020503 T4342 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 2020508 T4378 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 2020523 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2020533 T4394 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 2020535 T4447 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 2020536 T4447 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> ASYNC NEW_CORE C1976 name=collection1 org.apache.solr.core.SolrCore@13df4bf7 url=http://127.0.0.1:50490/collection1 node=127.0.0.1:50490_ C1976_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:50490_, base_url=http://127.0.0.1:50490}
[junit4:junit4] 2> 2021096 T4432 C1976 P50490 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:50480/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 2021097 T4432 C1976 P50490 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:50490 START replicas=[http://127.0.0.1:50480/collection1/] nUpdates=100
[junit4:junit4] 2> 2021098 T4432 C1976 P50490 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 2021099 T4432 C1976 P50490 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 2021099 T4432 C1976 P50490 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 2021099 T4432 C1976 P50490 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 2021100 T4432 C1976 P50490 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 2021100 T4432 C1976 P50490 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:50480/collection1/. core=collection1
[junit4:junit4] 2> 2021101 T4432 C1976 P50490 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C1977 name=collection1 org.apache.solr.core.SolrCore@27e90511 url=http://127.0.0.1:50480/collection1 node=127.0.0.1:50480_ C1977_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50480_, base_url=http://127.0.0.1:50480, leader=true}
[junit4:junit4] 2> 2021146 T4372 C1977 P50480 oasc.SolrCore.execute [collection1] webapp= path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0
[junit4:junit4] 2> 2021147 T4373 C1977 P50480 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 2021155 T4373 C1977 P50480 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty2/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 2021156 T4373 C1977 P50480 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2021161 T4373 C1977 P50480 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty2/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty2/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 2021162 T4373 C1977 P50480 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 2021166 T4373 C1977 P50480 oass.SolrIndexSearcher.<init> Opening Searcher@5fb5ce53 realtime
[junit4:junit4] 2> 2021167 T4373 C1977 P50480 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 2021168 T4373 C1977 P50480 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 21
[junit4:junit4] 2> 2021170 T4432 C1976 P50490 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 2021171 T4432 C1976 P50490 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 2021179 T4367 C1977 P50480 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 2021181 T4367 C1977 P50480 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=4
[junit4:junit4] 2> 2021182 T4432 C1976 P50490 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 2021183 T4432 C1976 P50490 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 2021183 T4432 C1976 P50490 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 2021187 T4371 C1977 P50480 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=1
[junit4:junit4] 2> 2021189 T4432 C1976 P50490 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 2021191 T4432 C1976 P50490 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty5/index.20130615015359984
[junit4:junit4] 2> 2021192 T4432 C1976 P50490 oash.SnapPuller.fetchLatestIndex Starting download to org.apache.lucene.store.NIOFSDirectory@/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty5/index.20130615015359984 lockFactory=org.apache.lucene.store.NativeFSLockFactory@694339d5 fullCopy=false
[junit4:junit4] 2> 2021197 T4369 C1977 P50480 oasc.SolrCore.execute [collection1] webapp= path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=1
[junit4:junit4] 2> 2021200 T4432 C1976 P50490 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 2021205 T4432 C1976 P50490 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 2021205 T4432 C1976 P50490 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 2021213 T4432 C1976 P50490 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty5/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty5/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 2021215 T4432 C1976 P50490 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 2021217 T4432 C1976 P50490 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 2021221 T4432 C1976 P50490 oass.SolrIndexSearcher.<init> Opening Searcher@60543a88 main
[junit4:junit4] 2> 2021224 T4431 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@60543a88 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 2021225 T4432 C1976 P50490 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty5/index.20130615015359984 [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty5/index.20130615015359984;done=true>>]
[junit4:junit4] 2> 2021226 T4432 C1976 P50490 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty5/index.20130615015359984
[junit4:junit4] 2> 2021226 T4432 C1976 P50490 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty5/index.20130615015359984
[junit4:junit4] 2> 2021229 T4432 C1976 P50490 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 2021229 T4432 C1976 P50490 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 2021229 T4432 C1976 P50490 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 2021230 T4432 C1976 P50490 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2021233 T4432 C1976 P50490 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 2021546 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2021548 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"7",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50497_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50497"}
[junit4:junit4] 2> 2021549 T4343 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4] 2> 2021549 T4343 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard3
[junit4:junit4] 2> 2021560 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"6",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50490_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50490"}
[junit4:junit4] 2> 2021572 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2021572 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2021573 T4427 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2021574 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2021572 T4446 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2021574 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2021574 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2021574 T4410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2022546 T4447 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371254036894/collection1
[junit4:junit4] 2> 2022547 T4447 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 2022549 T4447 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 2022549 T4447 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 2022552 T4447 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371254036894/collection1/'
[junit4:junit4] 2> 2022556 T4447 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371254036894/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 2022562 T4447 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371254036894/collection1/lib/README' to classloader
[junit4:junit4] 2> 2022692 T4447 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 2022787 T4447 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 2022790 T4447 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 2022798 T4447 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 2023992 T4447 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 2024013 T4447 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 2024020 T4447 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 2024075 T4447 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2024082 T4447 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2024090 T4447 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 2024094 T4447 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 2024095 T4447 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 2024095 T4447 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 2024097 T4447 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 2024098 T4447 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 2024098 T4447 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 2024098 T4447 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371254036894/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty6/
[junit4:junit4] 2> 2024099 T4447 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@269f046
[junit4:junit4] 2> 2024099 T4447 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 2024100 T4447 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty6
[junit4:junit4] 2> 2024101 T4447 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty6/index/
[junit4:junit4] 2> 2024102 T4447 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty6/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 2024103 T4447 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty6/index
[junit4:junit4] 2> 2024112 T4447 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty6/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 2024113 T4447 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2024124 T4447 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 2024125 T4447 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 2024126 T4447 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 2024127 T4447 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 2024128 T4447 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 2024129 T4447 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 2024139 T4447 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 2024144 T4447 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 2024145 T4447 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 2024165 T4447 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 2024186 T4447 oass.SolrIndexSearcher.<init> Opening Searcher@438ebf72 main
[junit4:junit4] 2> 2024188 T4447 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 2024189 T4447 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 2024194 T4450 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@438ebf72 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 2024198 T4447 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 2024198 T4447 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50497 collection:collection1 shard:shard3
[junit4:junit4] 2> 2024207 T4447 oasc.ZkController.register We are http://127.0.0.1:50497/collection1/ and leader is http://127.0.0.1:50483/collection1/
[junit4:junit4] 2> 2024207 T4447 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50497
[junit4:junit4] 2> 2024208 T4447 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 2024208 T4447 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C1978 name=collection1 org.apache.solr.core.SolrCore@692694dc url=http://127.0.0.1:50497/collection1 node=127.0.0.1:50497_ C1978_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:50497_, base_url=http://127.0.0.1:50497}
[junit4:junit4] 2> 2024209 T4451 C1978 P50497 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 2024210 T4451 C1978 P50497 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 2024210 T4451 C1978 P50497 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 2024211 T4451 C1978 P50497 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2024213 T4447 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2024215 T4451 C1978 P50497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2024219 T4320 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 2024220 T4320 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 2024220 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2024250 T4389 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 7, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 2024606 T4343 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2024609 T4343 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"7",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard3",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:50497_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:50497"}
[junit4:junit4] 2> 2024622 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2024623 T4410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2024623 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2024624 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2024624 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2024623 T4427 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2024622 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2024623 T4446 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2024872 T4320 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 2024883 T4320 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50505
[junit4:junit4] 2> 2024885 T4320 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 2024886 T4320 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 2024888 T4320 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371254043040
[junit4:junit4] 2> 2024889 T4320 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371254043040/solr.xml
[junit4:junit4] 2> 2024890 T4320 oasc.CoreContainer.<init> New CoreContainer 1652290806
[junit4:junit4] 2> 2024892 T4320 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371254043040/'
[junit4:junit4] 2> 2024893 T4320 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371254043040/'
[junit4:junit4] 2> 2025161 T4320 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 2025162 T4320 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 2025163 T4320 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 2025164 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 2025165 T4320 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 2025165 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 2025166 T4320 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 2025166 T4320 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 2025167 T4320 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 2025168 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 2025194 T4320 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 2025195 T4320 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50470/solr
[junit4:junit4] 2> 2025196 T4320 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 2025198 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2025205 T4464 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@43b3c67d name:ZooKeeperConnection Watcher:127.0.0.1:50470 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2025206 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2025212 T4320 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 2025236 T4320 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2025243 T4466 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64cd7d0 name:ZooKeeperConnection Watcher:127.0.0.1:50470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2025244 T4320 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2025253 T4389 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 7, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 2025255 T4389 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=7&state=recovering&nodeName=127.0.0.1:50497_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1005
[junit4:junit4] 2> 2025267 T4320 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 2026283 T4320 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50505_
[junit4:junit4] 2> 2026288 T4320 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50505_
[junit4:junit4] 2> 2026297 T4349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2026303 T4342 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2026307 T4394 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 2026308 T4427 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 2026309 T4427 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4] 2> 2026313 T4446 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 2026314 T4446 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4] 2> 2026320 T4410 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 2026321 T4410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4] 2> 2026322 T4378 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 2026322 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4] 2> 2026325 T4466 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 2026327 T4362 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 2026328 T4362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4] 2> 2026336 T4394 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 2026337 T4349 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 2026338 T4342 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 2026357 T4467 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 2026358 T4467 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> ASYNC NEW_CORE C1979 name=collection1 org.apache.solr.core.SolrCore@692694dc url=http://127.0.0.1:50497/collection1 node=127.0.0.1:50497_ C1979_STATE=coll:collection1 core:collection1 props:{shard=shard3, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:50497_, base_url=http://127.0.0.1:50497}
[junit4:junit4] 2> 2027259 T4451 C1979 P50497 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:50483/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 2027259 T4451 C1979 P50497 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:50497 START replicas=[http://127.0.0.1:50483/collection1/] nUpdates=100
[junit4:junit4] 2> 2027260 T4451 C1979 P50497 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 2027260 T4451 C1979 P50497 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 2027260 T4451 C1979 P50497 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 2027260 T4451 C1979 P50497 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 2027261 T4451 C1979 P50497 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 2027261 T4451 C1979 P50497 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:50483/collection1/. core=collection1
[junit4:junit4] 2> 2027261 T4451 C1979 P50497 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C1980 name=collection1 org.apache.solr.core.SolrCore@10e04bf4 url=http://127.0.0.1:50483/collection1 node=127.0.0.1:50483_ C1980_STATE=coll:collection1 core:collection1 props:{shard=shard3, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:50483_, base_url=http://127.0.0.1:50483, leader=true}
[junit4:junit4] 2> 2027302 T4389 C1980 P50483 oasc.SolrCore.execute [collection1] webapp= path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=1
[junit4:junit4] 2> 2027306 T4387 C1980 P50483 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 2027316 T4387 C1980 P50483 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty3/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 2027317 T4387 C1980 P50483 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2027323 T4387 C1980 P50483 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty3/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty3/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 2027325 T4387 C1980 P50483 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 2027329 T4387 C1980 P50483 oass.SolrIndexSearcher.<init> Opening Searcher@18ea2e8 realtime
[junit4:junit4] 2> 2027331 T4387 C1980 P50483 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 2027333 T4387 C1980 P50483 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 27
[junit4:junit4] 2> 2027338 T4451 C1979 P50497 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 2027338 T4451 C1979 P50497 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 2027343 T4390 C1980 P50483 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 2027344 T4390 C1980 P50483 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=2
[junit4:junit4] 2> 2027346 T4451 C1979 P50497 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 2027346 T4451 C1979 P50497 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 2027347 T4451 C1979 P50497 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 2027354 T4385 C1980 P50483 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=4
[junit4:junit4] 2> 2027356 T4451 C1979 P50497 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 2027358 T4451 C1979 P50497 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty6/index.20130615015406152
[junit4:junit4] 2> 2027360 T4451 C1979 P50497 oash.SnapPuller.fetchLatestIndex Starting download to org.apache.lucene.store.NIOFSDirectory@/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty6/index.20130615015406152 lockFactory=org.apache.lucene.store.NativeFSLockFactory@39d668f2 fullCopy=false
[junit4:junit4] 2> 2027365 T4386 C1980 P50483 oasc.SolrCore.execute [collection1] webapp= path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0
[junit4:junit4] 2> 2027369 T4451 C1979 P50497 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 2027373 T4451 C1979 P50497 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 2027374 T4451 C1979 P50497 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 2027383 T4451 C1979 P50497 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty6/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty6/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 2027384 T4451 C1979 P50497 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 2027385 T4451 C1979 P50497 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 2027387 T4451 C1979 P50497 oass.SolrIndexSearcher.<init> Opening Searcher@2108a7a4 main
[junit4:junit4] 2> 2027389 T4450 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2108a7a4 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 2027390 T4451 C1979 P50497 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty6/index.20130615015406152 [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty6/index.20130615015406152;done=true>>]
[junit4:junit4] 2> 2027390 T4451 C1979 P50497 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371254002309/jetty6/ind
[...truncated too long message...]
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
[junit4:junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
[junit4:junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
[junit4:junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
[junit4:junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
[junit4:junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
[junit4:junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4:junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
[junit4:junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4:junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
[junit4:junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
[junit4:junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
[junit4:junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
[junit4:junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4:junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
[junit4:junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
[junit4:junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
[junit4:junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
[junit4:junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4:junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
[junit4:junit4] 2> at java.lang.Thread.run(Thread.java:722)
[junit4:junit4] 2> 59) Thread[id=4602, name=Thread-1947-SendThread(localhost:50470), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
[junit4:junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4:junit4] 2> at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:86)
[junit4:junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:937)
[junit4:junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:993)
[junit4:junit4] 2> 9212802 T4602 oaz.ClientCnxn$SendThread.run WARN Session 0x13f451b9298001e for server null, unexpected error, closing socket connection and attempting reconnect java.net.ConnectException: Connection refused
[junit4:junit4] 2> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
[junit4:junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:692)
[junit4:junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:350)
[junit4:junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
[junit4:junit4] 2>
[junit4:junit4] 2> NOTE: test params are: codec=Lucene42: {range_facet_l=Lucene41(blocksize=128), text=PostingsFormat(name=MockSep), _version_=Lucene41(blocksize=128), rnd_b=PostingsFormat(name=Lucene41WithOrds), intDefault=Lucene41(blocksize=128), timestamp=Lucene41(blocksize=128), id=PostingsFormat(name=Lucene41WithOrds), a_t=Lucene41(blocksize=128), range_facet_sl=PostingsFormat(name=MockSep), range_facet_si=Pulsing41(freqCutoff=14 minBlockSize=89 maxBlockSize=184), other_tl1=Lucene41(blocksize=128), multiDefault=Pulsing41(freqCutoff=14 minBlockSize=89 maxBlockSize=184), a_si=Pulsing41(freqCutoff=14 minBlockSize=89 maxBlockSize=184)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=sv_SE, timezone=Europe/Sarajevo
[junit4:junit4] 2> NOTE: Mac OS X 10.8.4 x86_64/Oracle Corporation 1.7.0_21 (64-bit)/cpus=2,threads=69,free=167315440,total=272470016
[junit4:junit4] 2> NOTE: All tests run in this JVM: [TestCopyFieldCollectionResource, TestRealTimeGet, UpdateParamsTest, MBeansHandlerTest, TestUniqueKeyFieldResource, TestSchemaVersionResource, IndexSchemaTest, TestSolr4Spatial, TestSolrXml, TestManagedSchemaFieldResource, LeaderElectionIntegrationTest, TestBadConfig, FieldMutatingUpdateProcessorTest, SyncSliceTest, MultiTermTest, TestSolrXMLSerializer, TestDistributedGrouping, SOLR749Test, PathHierarchyTokenizerFactoryTest, TestReloadAndDeleteDocs, CollectionsAPIDistributedZkTest, QueryElevationComponentTest, MoreLikeThisHandlerTest, SpellCheckCollatorTest, TestAnalyzedSuggestions, SolrCmdDistributorTest, TestWriterPerf, TestZkChroot, FieldAnalysisRequestHandlerTest, TestStressLucene, DocumentAnalysisRequestHandlerTest, TestConfig, TestRTGBase, TestFieldTypeResource, TestBinaryField, NotRequiredUniqueKeyTest, TestCollationField, TestSolrIndexConfig, TestLMDirichletSimilarityFactory, DOMUtilTest, TestRemoteStreaming, TestStressReorder, SpatialFilterTest, TestJmxMonitoredMap, TestQueryTypes, ChaosMonkeyNothingIsSafeTest, PolyFieldTest, DirectUpdateHandlerTest, CSVRequestHandlerTest, SolrCoreTest, TestCoreContainer, TestQuerySenderNoQuery, CachingDirectoryFactoryTest, TestShardHandlerFactory, SuggesterFSTTest, TestFaceting, TestQuerySenderListener, TestComponentsName, DefaultValueUpdateProcessorTest, HighlighterConfigTest, CurrencyFieldOpenExchangeTest, SuggesterTSTTest, CoreContainerCoreInitFailuresTest, SpellCheckComponentTest, TestSolrDeletionPolicy1, SimpleFacetsTest, SoftAutoCommitTest, DirectUpdateHandlerOptimizeTest, XmlUpdateRequestHandlerTest, SearchHandlerTest, TestSolrQueryParserResource, SolrInfoMBeanTest, OverseerTest, TestPropInject, TestIBSimilarityFactory, TestXIncludeConfig, TestUpdate, DistributedSpellCheckComponentTest, TestDFRSimilarityFactory, TestValueSourceCache, TestFastWriter, TestPerFieldSimilarity, TestJoin, OverseerCollectionProcessorTest, TestRandomFaceting, ConvertedLegacyTest, TestDocumentBuilder, FileBasedSpellCheckerTest, PingRequestHandlerTest, NoCacheHeaderTest, BadCopyFieldTest, TestExtendedDismaxParser, SpellingQueryConverterTest, ZkControllerTest, DistributedTermsComponentTest, TestPropInjectDefaults, BinaryUpdateRequestHandlerTest, ShardRoutingTest, ZkSolrClientTest, IndexSchemaRuntimeFieldTest, TestCodecSupport, CacheHeaderTest, PluginInfoTest, TestReplicationHandler, ResponseLogComponentTest, DateFieldTest, TestSuggestSpellingConverter, CopyFieldTest, SuggesterWFSTTest, HardAutoCommitTest, TestPostingsSolrHighlighter, BasicZkTest, TestCoreDiscovery, ScriptEngineTest, TestLMJelinekMercerSimilarityFactory, TestMultiCoreConfBootstrap, TestPHPSerializedResponseWriter, TestReversedWildcardFilterFactory, TestFieldTypeCollectionResource, TestDocSet, IndexBasedSpellCheckerTest, DirectSolrSpellCheckerTest, TestHashPartitioner, EchoParamsTest, BasicFunctionalityTest, DirectSolrConnectionTest, TestDynamicFieldResource, TestMaxScoreQueryParser, ZkNodePropsTest, PeerSyncTest, ZkCLITest, DebugComponentTest, ClusterStateTest, TestSolrQueryParser, TestFoldingMultitermQuery, NumericFieldsTest, TestCSVLoader, TestSerializedLuceneMatchVersion, TestFastLRUCache, TestDistributedSearch, TestLRUCache, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest]
[junit4:junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.seed=F3DD7ACAA9FDF51A -Dtests.slow=true -Dtests.locale=sv_SE -Dtests.timezone=Europe/Sarajevo -Dtests.file.encoding=US-ASCII
[junit4:junit4] ERROR 0.00s | ChaosMonkeySafeLeaderTest (suite) <<<
[junit4:junit4] > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
[junit4:junit4] > at __randomizedtesting.SeedInfo.seed([F3DD7ACAA9FDF51A]:0)
[junit4:junit4] Completed in 7229.36s, 1 test, 2 errors <<< FAILURES!
[...truncated 448 lines...]
BUILD FAILED
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:385: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:365: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:39: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:181: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:437: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1240: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:884: There were test failures: 296 suites, 1243 tests, 1 suite-level error, 1 error, 586 ignored (2 assumptions)
Total time: 202 minutes 4 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure
[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.7.0) - Build # 561 -
Still Failing!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/561/
Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseG1GC
1 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
Error Message:
ERROR: SolrIndexSearcher opens=44 closes=43
Stack Trace:
java.lang.AssertionError: ERROR: SolrIndexSearcher opens=44 closes=43
at __randomizedtesting.SeedInfo.seed([B9A2CFAB6B3FFFF4]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:254)
at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:102)
at sun.reflect.GeneratedMethodAccessor24.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:601)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:700)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:722)
Build Log:
[...truncated 9524 lines...]
[junit4:junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
[junit4:junit4] 2> 1420698 T2682 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /doxo/ht
[junit4:junit4] 2> 1420709 T2682 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4:junit4] 2> Creating dataDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./solrtest-ChaosMonkeySafeLeaderTest-1371226275506
[junit4:junit4] 2> 1420712 T2682 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4:junit4] 2> 1420714 T2683 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4:junit4] 2> 1420815 T2682 oasc.ZkTestServer.run start zk server on port:49646
[junit4:junit4] 2> 1420821 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1420837 T2689 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@57162cc1 name:ZooKeeperConnection Watcher:127.0.0.1:49646 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1420838 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1420839 T2682 oascc.SolrZkClient.makePath makePath: /solr
[junit4:junit4] 2> 1420861 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1420871 T2691 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62500101 name:ZooKeeperConnection Watcher:127.0.0.1:49646/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1420871 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1420873 T2682 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4:junit4] 2> 1420891 T2682 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4:junit4] 2> 1420903 T2682 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4:junit4] 2> 1420912 T2682 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4:junit4] 2> 1420924 T2682 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 1420926 T2682 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 1420943 T2682 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4:junit4] 2> 1420944 T2682 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4:junit4] 2> 1420953 T2682 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4:junit4] 2> 1420954 T2682 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4:junit4] 2> 1420981 T2682 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4:junit4] 2> 1420983 T2682 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4:junit4] 2> 1420996 T2682 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4:junit4] 2> 1420998 T2682 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4:junit4] 2> 1421011 T2682 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 1421013 T2682 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 1421023 T2682 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 1421024 T2682 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 1421035 T2682 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 1421037 T2682 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 1421048 T2682 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4:junit4] 2> 1421050 T2682 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4:junit4] 2> 1421726 T2682 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1421744 T2682 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49649
[junit4:junit4] 2> 1421746 T2682 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1421747 T2682 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1421748 T2682 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371226275866
[junit4:junit4] 2> 1421749 T2682 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371226275866/solr.xml
[junit4:junit4] 2> 1421749 T2682 oasc.CoreContainer.<init> New CoreContainer 1629750935
[junit4:junit4] 2> 1421750 T2682 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371226275866/'
[junit4:junit4] 2> 1421751 T2682 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371226275866/'
[junit4:junit4] 2> 1422159 T2682 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1422169 T2682 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1422173 T2682 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1422175 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1422178 T2682 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1422179 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1422179 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1422180 T2682 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1422181 T2682 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1422182 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1422224 T2682 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1422226 T2682 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49646/solr
[junit4:junit4] 2> 1422228 T2682 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1422240 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1422244 T2702 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ed23c56 name:ZooKeeperConnection Watcher:127.0.0.1:49646 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1422247 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1422262 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1422301 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1422325 T2704 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c8f9e23 name:ZooKeeperConnection Watcher:127.0.0.1:49646/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1422326 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1422339 T2682 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4:junit4] 2> 1422356 T2682 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4:junit4] 2> 1422373 T2682 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4:junit4] 2> 1422382 T2682 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49649_doxo%2Fht
[junit4:junit4] 2> 1422385 T2682 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49649_doxo%2Fht
[junit4:junit4] 2> 1422399 T2682 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4:junit4] 2> 1422430 T2682 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4:junit4] 2> 1422442 T2682 oasc.Overseer.start Overseer (id=89864685192413187-127.0.0.1:49649_doxo%2Fht-n_0000000000) starting
[junit4:junit4] 2> 1422454 T2682 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4:junit4] 2> 1422468 T2706 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4:junit4] 2> 1422475 T2682 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4:junit4] 2> 1422484 T2682 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4:junit4] 2> 1422492 T2682 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1422535 T2705 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4:junit4] 2> 1422539 T2707 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1422540 T2707 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1422551 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1422554 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49649_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49649/doxo/ht"}
[junit4:junit4] 2> 1422555 T2705 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
[junit4:junit4] 2> 1422556 T2705 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1422575 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1423549 T2707 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371226275866/collection1
[junit4:junit4] 2> 1423549 T2707 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4:junit4] 2> 1423551 T2707 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1423551 T2707 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4:junit4] 2> 1423555 T2707 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371226275866/collection1/'
[junit4:junit4] 2> 1423557 T2707 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371226275866/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1423557 T2707 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371226275866/collection1/lib/README' to classloader
[junit4:junit4] 2> 1423644 T2707 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1423793 T2707 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1423798 T2707 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1423808 T2707 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1425181 T2707 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1425197 T2707 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1425206 T2707 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1425273 T2707 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1425287 T2707 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1425311 T2707 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1425322 T2707 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1425324 T2707 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1425325 T2707 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1425330 T2707 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1425332 T2707 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1425333 T2707 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1425334 T2707 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1371226275866/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/control/data/
[junit4:junit4] 2> 1425335 T2707 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f3c2edc
[junit4:junit4] 2> 1425337 T2707 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1425340 T2707 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/control/data
[junit4:junit4] 2> 1425342 T2707 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/control/data/index/
[junit4:junit4] 2> 1425345 T2707 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/control/data/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1425347 T2707 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/control/data/index
[junit4:junit4] 2> 1425361 T2707 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/control/data/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1425363 T2707 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1425373 T2707 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1425374 T2707 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1425376 T2707 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1425379 T2707 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1425382 T2707 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1425383 T2707 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1425384 T2707 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1425385 T2707 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1425387 T2707 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1425422 T2707 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1425445 T2707 oass.SolrIndexSearcher.<init> Opening Searcher@1c01a61a main
[junit4:junit4] 2> 1425447 T2707 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1425448 T2707 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1425454 T2708 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1c01a61a main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1425459 T2707 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1425460 T2707 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49649/doxo/ht collection:control_collection shard:shard1
[junit4:junit4] 2> 1425465 T2707 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4:junit4] 2> 1425516 T2707 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 1425536 T2707 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 1425538 T2707 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 1425538 T2707 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49649/doxo/ht/collection1/
[junit4:junit4] 2> 1425540 T2707 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 1425541 T2707 oasc.SyncStrategy.syncToMe http://127.0.0.1:49649/doxo/ht/collection1/ has no replicas
[junit4:junit4] 2> 1425542 T2707 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49649/doxo/ht/collection1/
[junit4:junit4] 2> 1425543 T2707 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4:junit4] 2> 1425612 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1425661 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1425694 T2707 oasc.ZkController.register We are http://127.0.0.1:49649/doxo/ht/collection1/ and leader is http://127.0.0.1:49649/doxo/ht/collection1/
[junit4:junit4] 2> 1425695 T2707 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49649/doxo/ht
[junit4:junit4] 2> 1425695 T2707 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 1425695 T2707 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1425696 T2707 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1425704 T2707 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1425709 T2682 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1425710 T2682 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1425712 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1425755 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:
[junit4:junit4] 2> 1425759 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1425776 T2711 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@214c6507 name:ZooKeeperConnection Watcher:127.0.0.1:49646/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1425777 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1425788 T2682 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1425799 T2682 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
[junit4:junit4] 2> 1426446 T2682 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1426462 T2682 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49653
[junit4:junit4] 2> 1426471 T2682 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1426473 T2682 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1426474 T2682 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371226280596
[junit4:junit4] 2> 1426474 T2682 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371226280596/solr.xml
[junit4:junit4] 2> 1426475 T2682 oasc.CoreContainer.<init> New CoreContainer 779944513
[junit4:junit4] 2> 1426477 T2682 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371226280596/'
[junit4:junit4] 2> 1426478 T2682 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371226280596/'
[junit4:junit4] 2> 1426767 T2682 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1426769 T2682 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1426771 T2682 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1426772 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1426774 T2682 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1426775 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1426776 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1426776 T2682 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1426777 T2682 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1426778 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1426818 T2682 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1426823 T2682 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49646/solr
[junit4:junit4] 2> 1426824 T2682 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1426833 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1426844 T2722 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6103784b name:ZooKeeperConnection Watcher:127.0.0.1:49646 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1426845 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1426862 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1426895 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1426901 T2724 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@bf6f88d name:ZooKeeperConnection Watcher:127.0.0.1:49646/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1426902 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1426922 T2682 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1427194 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1427197 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49649_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49649/doxo/ht"}
[junit4:junit4] 2> 1427225 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1427225 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1427225 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1427949 T2682 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49653_doxo%2Fht
[junit4:junit4] 2> 1427960 T2682 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49653_doxo%2Fht
[junit4:junit4] 2> 1427977 T2711 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 1427981 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1427982 T2724 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 1427982 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1427984 T2704 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 1427985 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1428024 T2725 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1428024 T2725 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1428755 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1428758 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49653_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49653/doxo/ht"}
[junit4:junit4] 2> 1428760 T2705 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 3
[junit4:junit4] 2> 1428761 T2705 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1428780 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1428780 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1428781 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1429031 T2725 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371226280596/collection1
[junit4:junit4] 2> 1429032 T2725 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1429036 T2725 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1429037 T2725 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1429047 T2725 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371226280596/collection1/'
[junit4:junit4] 2> 1429052 T2725 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371226280596/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1429056 T2725 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371226280596/collection1/lib/README' to classloader
[junit4:junit4] 2> 1429192 T2725 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1429342 T2725 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1429347 T2725 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1429366 T2725 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1430816 T2725 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1430840 T2725 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1430848 T2725 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1430913 T2725 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1430929 T2725 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1430944 T2725 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1430949 T2725 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1430951 T2725 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1430951 T2725 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1430956 T2725 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1430956 T2725 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1430957 T2725 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1430957 T2725 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1371226280596/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty1/
[junit4:junit4] 2> 1430958 T2725 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f3c2edc
[junit4:junit4] 2> 1430959 T2725 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1430961 T2725 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty1
[junit4:junit4] 2> 1430962 T2725 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty1/index/
[junit4:junit4] 2> 1430964 T2725 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty1/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1430967 T2725 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty1/index
[junit4:junit4] 2> 1430988 T2725 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty1/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1430989 T2725 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1430996 T2725 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1430998 T2725 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1431000 T2725 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1431002 T2725 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1431003 T2725 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1431004 T2725 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1431004 T2725 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1431006 T2725 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1431008 T2725 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1431033 T2725 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1431052 T2725 oass.SolrIndexSearcher.<init> Opening Searcher@55c8477 main
[junit4:junit4] 2> 1431054 T2725 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1431055 T2725 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1431069 T2726 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@55c8477 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1431072 T2725 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1431072 T2725 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49653/doxo/ht collection:collection1 shard:shard1
[junit4:junit4] 2> 1431074 T2725 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4:junit4] 2> 1431101 T2725 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 1431113 T2725 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 1431114 T2725 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 1431115 T2725 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49653/doxo/ht/collection1/
[junit4:junit4] 2> 1431115 T2725 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 1431116 T2725 oasc.SyncStrategy.syncToMe http://127.0.0.1:49653/doxo/ht/collection1/ has no replicas
[junit4:junit4] 2> 1431117 T2725 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49653/doxo/ht/collection1/
[junit4:junit4] 2> 1431117 T2725 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4:junit4] 2> 1431819 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1431850 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1431855 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1431856 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1431865 T2725 oasc.ZkController.register We are http://127.0.0.1:49653/doxo/ht/collection1/ and leader is http://127.0.0.1:49653/doxo/ht/collection1/
[junit4:junit4] 2> 1431866 T2725 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49653/doxo/ht
[junit4:junit4] 2> 1431866 T2725 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 1431867 T2725 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1431867 T2725 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1431880 T2725 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1431888 T2682 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1431889 T2682 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1431890 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1432832 T2682 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1432848 T2682 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49656
[junit4:junit4] 2> 1432850 T2682 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1432852 T2682 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1432856 T2682 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371226286728
[junit4:junit4] 2> 1432858 T2682 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371226286728/solr.xml
[junit4:junit4] 2> 1432862 T2682 oasc.CoreContainer.<init> New CoreContainer 400571813
[junit4:junit4] 2> 1432864 T2682 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371226286728/'
[junit4:junit4] 2> 1432866 T2682 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371226286728/'
[junit4:junit4] 2> 1433886 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1433899 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49653_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49653/doxo/ht"}
[junit4:junit4] 2> 1433942 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1433943 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1433942 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1433971 T2682 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1433973 T2682 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1433976 T2682 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1433977 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1433987 T2682 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1433991 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1434293 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1434294 T2682 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1434294 T2682 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1434295 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1434319 T2682 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1434320 T2682 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49646/solr
[junit4:junit4] 2> 1434320 T2682 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1434322 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1434328 T2738 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@61397418 name:ZooKeeperConnection Watcher:127.0.0.1:49646 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1434330 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1434337 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1434377 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1434383 T2740 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5885804e name:ZooKeeperConnection Watcher:127.0.0.1:49646/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1434384 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1434443 T2682 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1435463 T2682 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49656_doxo%2Fht
[junit4:junit4] 2> 1435468 T2682 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49656_doxo%2Fht
[junit4:junit4] 2> 1435488 T2740 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1435489 T2704 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1435490 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1435489 T2711 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1435491 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1435491 T2724 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1435492 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1435536 T2741 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1435537 T2741 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1437020 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1437022 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49656_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49656/doxo/ht"}
[junit4:junit4] 2> 1437023 T2705 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4] 2> 1437023 T2705 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 1437036 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1437036 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1437037 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1437037 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1437579 T2741 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371226286728/collection1
[junit4:junit4] 2> 1437580 T2741 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1437583 T2741 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1437583 T2741 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1437588 T2741 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371226286728/collection1/'
[junit4:junit4] 2> 1437590 T2741 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371226286728/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1437590 T2741 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371226286728/collection1/lib/README' to classloader
[junit4:junit4] 2> 1437677 T2741 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1437805 T2741 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1437811 T2741 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1437820 T2741 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1439205 T2741 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1439219 T2741 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1439224 T2741 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1439266 T2741 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1439277 T2741 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1439290 T2741 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1439295 T2741 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1439296 T2741 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1439297 T2741 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1439302 T2741 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1439302 T2741 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1439303 T2741 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1439304 T2741 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1371226286728/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty2/
[junit4:junit4] 2> 1439305 T2741 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f3c2edc
[junit4:junit4] 2> 1439306 T2741 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1439308 T2741 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty2
[junit4:junit4] 2> 1439313 T2741 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty2/index/
[junit4:junit4] 2> 1439317 T2741 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty2/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1439319 T2741 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty2/index
[junit4:junit4] 2> 1439582 T2741 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty2/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1439583 T2741 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1439588 T2741 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1439589 T2741 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1439590 T2741 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1439591 T2741 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1439593 T2741 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1439593 T2741 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1439593 T2741 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1439595 T2741 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1439596 T2741 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1439627 T2741 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1439650 T2741 oass.SolrIndexSearcher.<init> Opening Searcher@560355df main
[junit4:junit4] 2> 1439653 T2741 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1439653 T2741 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1439661 T2742 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@560355df main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1439666 T2741 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1439666 T2741 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49656/doxo/ht collection:collection1 shard:shard2
[junit4:junit4] 2> 1439671 T2741 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4:junit4] 2> 1439701 T2741 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 1439746 T2741 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 1439747 T2741 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 1439747 T2741 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49656/doxo/ht/collection1/
[junit4:junit4] 2> 1439748 T2741 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 1439748 T2741 oasc.SyncStrategy.syncToMe http://127.0.0.1:49656/doxo/ht/collection1/ has no replicas
[junit4:junit4] 2> 1439749 T2741 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49656/doxo/ht/collection1/
[junit4:junit4] 2> 1439749 T2741 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4:junit4] 2> 1440073 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1440108 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1440109 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1440110 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1440109 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1440147 T2741 oasc.ZkController.register We are http://127.0.0.1:49656/doxo/ht/collection1/ and leader is http://127.0.0.1:49656/doxo/ht/collection1/
[junit4:junit4] 2> 1440147 T2741 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49656/doxo/ht
[junit4:junit4] 2> 1440148 T2741 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 1440148 T2741 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1440149 T2741 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1440164 T2741 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1440171 T2682 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1440172 T2682 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1440172 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1440866 T2682 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1440874 T2682 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49659
[junit4:junit4] 2> 1440877 T2682 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1440878 T2682 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1440879 T2682 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371226295006
[junit4:junit4] 2> 1440879 T2682 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371226295006/solr.xml
[junit4:junit4] 2> 1440880 T2682 oasc.CoreContainer.<init> New CoreContainer 344201984
[junit4:junit4] 2> 1440881 T2682 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371226295006/'
[junit4:junit4] 2> 1440881 T2682 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371226295006/'
[junit4:junit4] 2> 1441250 T2682 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1441251 T2682 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1441252 T2682 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1441252 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1441253 T2682 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1441254 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1441255 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1441255 T2682 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1441256 T2682 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1441257 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1441298 T2682 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1441300 T2682 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49646/solr
[junit4:junit4] 2> 1441301 T2682 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1441314 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1441323 T2754 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6db54b61 name:ZooKeeperConnection Watcher:127.0.0.1:49646 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1441325 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1441334 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1441361 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1441370 T2756 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7284852d name:ZooKeeperConnection Watcher:127.0.0.1:49646/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1441371 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1441389 T2682 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1441642 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1441645 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49656_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49656/doxo/ht"}
[junit4:junit4] 2> 1441667 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1441668 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1441669 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1441669 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1441677 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1442405 T2682 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49659_doxo%2Fht
[junit4:junit4] 2> 1442408 T2682 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49659_doxo%2Fht
[junit4:junit4] 2> 1442422 T2711 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1442422 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1442423 T2756 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1442424 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1442429 T2740 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1442433 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1442432 T2704 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1442434 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1442432 T2724 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1442435 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1442472 T2757 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1442473 T2757 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1443195 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1443200 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49659_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49659/doxo/ht"}
[junit4:junit4] 2> 1443200 T2705 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4] 2> 1443201 T2705 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard3
[junit4:junit4] 2> 1443226 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1443227 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1443227 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1443228 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1443228 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1443478 T2757 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371226295006/collection1
[junit4:junit4] 2> 1443479 T2757 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1443482 T2757 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1443483 T2757 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1443490 T2757 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371226295006/collection1/'
[junit4:junit4] 2> 1443493 T2757 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371226295006/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1443495 T2757 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371226295006/collection1/lib/README' to classloader
[junit4:junit4] 2> 1443654 T2757 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1443835 T2757 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1443843 T2757 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1443859 T2757 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1445508 T2757 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1445526 T2757 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1445534 T2757 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1445579 T2757 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1445597 T2757 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1445612 T2757 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1445617 T2757 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1445618 T2757 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1445619 T2757 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1445623 T2757 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1445624 T2757 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1445624 T2757 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1445625 T2757 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1371226295006/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty3/
[junit4:junit4] 2> 1445626 T2757 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f3c2edc
[junit4:junit4] 2> 1445627 T2757 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1445629 T2757 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty3
[junit4:junit4] 2> 1445631 T2757 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty3/index/
[junit4:junit4] 2> 1445632 T2757 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty3/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1445634 T2757 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty3/index
[junit4:junit4] 2> 1445644 T2757 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty3/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1445644 T2757 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1445652 T2757 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1445653 T2757 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1445656 T2757 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1445657 T2757 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1445659 T2757 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1445659 T2757 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1445660 T2757 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1445662 T2757 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1445663 T2757 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1445689 T2757 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1445699 T2757 oass.SolrIndexSearcher.<init> Opening Searcher@bc45581 main
[junit4:junit4] 2> 1445701 T2757 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1445701 T2757 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1445706 T2758 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@bc45581 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1445711 T2757 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1445711 T2757 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49659/doxo/ht collection:collection1 shard:shard3
[junit4:junit4] 2> 1445719 T2757 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
[junit4:junit4] 2> 1445740 T2757 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 1445748 T2757 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 1445748 T2757 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 1445748 T2757 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49659/doxo/ht/collection1/
[junit4:junit4] 2> 1445749 T2757 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 1445749 T2757 oasc.SyncStrategy.syncToMe http://127.0.0.1:49659/doxo/ht/collection1/ has no replicas
[junit4:junit4] 2> 1445749 T2757 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49659/doxo/ht/collection1/
[junit4:junit4] 2> 1445749 T2757 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
[junit4:junit4] 2> 1446258 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1446299 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1446299 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1446300 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1446300 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1446302 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1446328 T2757 oasc.ZkController.register We are http://127.0.0.1:49659/doxo/ht/collection1/ and leader is http://127.0.0.1:49659/doxo/ht/collection1/
[junit4:junit4] 2> 1446329 T2757 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49659/doxo/ht
[junit4:junit4] 2> 1446329 T2757 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 1446330 T2757 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1446330 T2757 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1446335 T2757 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1446339 T2682 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1446340 T2682 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1446342 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1447032 T2682 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1447044 T2682 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49662
[junit4:junit4] 2> 1447048 T2682 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1447049 T2682 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1447050 T2682 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371226301170
[junit4:junit4] 2> 1447051 T2682 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371226301170/solr.xml
[junit4:junit4] 2> 1447052 T2682 oasc.CoreContainer.<init> New CoreContainer 1022658122
[junit4:junit4] 2> 1447053 T2682 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371226301170/'
[junit4:junit4] 2> 1447054 T2682 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371226301170/'
[junit4:junit4] 2> 1447344 T2682 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1447345 T2682 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1447346 T2682 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1447347 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1447347 T2682 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1447348 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1447349 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1447350 T2682 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1447350 T2682 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1447351 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1447391 T2682 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1447395 T2682 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49646/solr
[junit4:junit4] 2> 1447397 T2682 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1447400 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1447407 T2770 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b57d258 name:ZooKeeperConnection Watcher:127.0.0.1:49646 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1447408 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1447416 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1447463 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1447466 T2772 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31894d1 name:ZooKeeperConnection Watcher:127.0.0.1:49646/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1447467 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1447487 T2682 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1447848 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1447850 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard3",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49659_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49659/doxo/ht"}
[junit4:junit4] 2> 1447871 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1447872 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1447873 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1447874 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1447876 T2772 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1447873 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1448505 T2682 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49662_doxo%2Fht
[junit4:junit4] 2> 1448508 T2682 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49662_doxo%2Fht
[junit4:junit4] 2> 1448516 T2772 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1448524 T2772 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1448545 T2704 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1448545 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1448549 T2740 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1448550 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1448552 T2711 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1448558 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1448563 T2756 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1448563 T2724 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1448564 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1448563 T2773 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1448565 T2773 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1448563 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1449406 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1449411 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49662_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49662/doxo/ht"}
[junit4:junit4] 2> 1449413 T2705 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4] 2> 1449414 T2705 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1449439 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1449448 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1449451 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1449450 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1449452 T2772 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1449451 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1449571 T2773 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371226301170/collection1
[junit4:junit4] 2> 1449572 T2773 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1449574 T2773 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1449575 T2773 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1449582 T2773 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371226301170/collection1/'
[junit4:junit4] 2> 1449591 T2773 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371226301170/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1449592 T2773 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371226301170/collection1/lib/README' to classloader
[junit4:junit4] 2> 1449700 T2773 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1449846 T2773 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1449852 T2773 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1449869 T2773 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1451302 T2773 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1451319 T2773 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1451326 T2773 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1451410 T2773 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1451421 T2773 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1451432 T2773 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1451437 T2773 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1451439 T2773 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1451439 T2773 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1451443 T2773 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1451444 T2773 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1451444 T2773 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1451445 T2773 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1371226301170/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty4/
[junit4:junit4] 2> 1451445 T2773 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f3c2edc
[junit4:junit4] 2> 1451446 T2773 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1451448 T2773 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty4
[junit4:junit4] 2> 1451448 T2773 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty4/index/
[junit4:junit4] 2> 1451449 T2773 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty4/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1451450 T2773 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty4/index
[junit4:junit4] 2> 1451457 T2773 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty4/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1451457 T2773 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1451461 T2773 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1451462 T2773 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1451463 T2773 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1451466 T2773 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1451472 T2773 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1451472 T2773 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1451473 T2773 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1451474 T2773 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1451475 T2773 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1451498 T2773 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1451512 T2773 oass.SolrIndexSearcher.<init> Opening Searcher@66cea91a main
[junit4:junit4] 2> 1451514 T2773 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1451515 T2773 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1451521 T2774 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@66cea91a main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1451534 T2773 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1451535 T2773 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49662/doxo/ht collection:collection1 shard:shard1
[junit4:junit4] 2> 1451544 T2773 oasc.ZkController.register We are http://127.0.0.1:49662/doxo/ht/collection1/ and leader is http://127.0.0.1:49653/doxo/ht/collection1/
[junit4:junit4] 2> 1451545 T2773 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49662/doxo/ht
[junit4:junit4] 2> 1451547 T2773 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 1451547 T2773 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C975 name=collection1 org.apache.solr.core.SolrCore@18b980ff url=http://127.0.0.1:49662/doxo/ht/collection1 node=127.0.0.1:49662_doxo%2Fht C975_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:49662_doxo%2Fht, base_url=http://127.0.0.1:49662/doxo/ht}
[junit4:junit4] 2> 1451549 T2775 C975 P49662 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 1451550 T2775 C975 P49662 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 1451550 T2775 C975 P49662 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 1451550 T2775 C975 P49662 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1451559 T2773 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1451560 T2775 C975 P49662 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1451595 T2682 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1451596 T2682 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1451597 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1451601 T2716 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 1452252 T2682 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1452261 T2682 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49666
[junit4:junit4] 2> 1452264 T2682 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1452265 T2682 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1452265 T2682 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371226306415
[junit4:junit4] 2> 1452266 T2682 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371226306415/solr.xml
[junit4:junit4] 2> 1452267 T2682 oasc.CoreContainer.<init> New CoreContainer 209460736
[junit4:junit4] 2> 1452269 T2682 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371226306415/'
[junit4:junit4] 2> 1452269 T2682 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371226306415/'
[junit4:junit4] 2> 1452495 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1452497 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49662_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49662/doxo/ht"}
[junit4:junit4] 2> 1452510 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1452511 T2772 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1452511 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1452512 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1452511 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1452515 T2682 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1452518 T2682 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1452519 T2682 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1452519 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1452520 T2682 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1452521 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1452521 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1452522 T2682 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1452523 T2682 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1452534 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1452512 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1452572 T2682 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1452578 T2682 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49646/solr
[junit4:junit4] 2> 1452578 T2682 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1452581 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1452590 T2787 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e545eb8 name:ZooKeeperConnection Watcher:127.0.0.1:49646 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1452592 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1452598 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1452604 T2716 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 1452606 T2716 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=5&state=recovering&nodeName=127.0.0.1:49662_doxo%252Fht&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1005
[junit4:junit4] 2> 1452629 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1452644 T2789 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d811999 name:ZooKeeperConnection Watcher:127.0.0.1:49646/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1452645 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1452670 T2682 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1453682 T2682 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49666_doxo%2Fht
[junit4:junit4] 2> 1453686 T2682 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49666_doxo%2Fht
[junit4:junit4] 2> 1453699 T2772 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1453708 T2704 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1453708 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1453709 T2711 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1453709 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1453710 T2740 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1453710 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1453709 T2724 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1453711 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1453710 T2789 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1453710 T2756 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1453713 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1453726 T2772 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1453746 T2790 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1453747 T2790 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1454051 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1454053 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"6",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49666_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49666/doxo/ht"}
[junit4:junit4] 2> 1454053 T2705 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4] 2> 1454054 T2705 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 1454068 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1454069 T2789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1454069 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1454069 T2772 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1454069 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1454069 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1454069 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> C975_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:49662_doxo%2Fht, base_url=http://127.0.0.1:49662/doxo/ht}
[junit4:junit4] 2> 1454613 T2775 C975 P49662 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:49653/doxo/ht/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 1454614 T2775 C975 P49662 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:49662/doxo/ht START replicas=[http://127.0.0.1:49653/doxo/ht/collection1/] nUpdates=100
[junit4:junit4] 2> 1454614 T2775 C975 P49662 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 1454617 T2775 C975 P49662 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 1454619 T2775 C975 P49662 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 1454619 T2775 C975 P49662 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 1454619 T2775 C975 P49662 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 1454619 T2775 C975 P49662 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:49653/doxo/ht/collection1/. core=collection1
[junit4:junit4] 2> 1454619 T2775 C975 P49662 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C976 name=collection1 org.apache.solr.core.SolrCore@58db97b3 url=http://127.0.0.1:49653/doxo/ht/collection1 node=127.0.0.1:49653_doxo%2Fht C976_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:49653_doxo%2Fht, base_url=http://127.0.0.1:49653/doxo/ht, leader=true}
[junit4:junit4] 2> 1454643 T2716 C976 P49653 oasc.SolrCore.execute [collection1] webapp=/doxo/ht path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=1
[junit4:junit4] 2> 1454654 T2720 C976 P49653 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 1454662 T2720 C976 P49653 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty1/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1454664 T2720 C976 P49653 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1454668 T2720 C976 P49653 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty1/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty1/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 1454669 T2720 C976 P49653 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1454672 T2720 C976 P49653 oass.SolrIndexSearcher.<init> Opening Searcher@b8f4304 realtime
[junit4:junit4] 2> 1454673 T2720 C976 P49653 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1454674 T2720 C976 P49653 oasup.LogUpdateProcessor.finish [collection1] webapp=/doxo/ht path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 20
[junit4:junit4] 2> 1454685 T2775 C975 P49662 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1454686 T2775 C975 P49662 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 1454693 T2717 C976 P49653 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1454694 T2717 C976 P49653 oasc.SolrCore.execute [collection1] webapp=/doxo/ht path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=4
[junit4:junit4] 2> 1454696 T2775 C975 P49662 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 1454696 T2775 C975 P49662 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 1454696 T2775 C975 P49662 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 1454706 T2718 C976 P49653 oasc.SolrCore.execute [collection1] webapp=/doxo/ht path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=2
[junit4:junit4] 2> 1454708 T2775 C975 P49662 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 1454710 T2775 C975 P49662 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty4/index.20130614171149503
[junit4:junit4] 2> 1454711 T2775 C975 P49662 oash.SnapPuller.fetchLatestIndex Starting download to org.apache.lucene.store.NIOFSDirectory@/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty4/index.20130614171149503 lockFactory=org.apache.lucene.store.NativeFSLockFactory@77a10c6e fullCopy=false
[junit4:junit4] 2> 1454716 T2719 C976 P49653 oasc.SolrCore.execute [collection1] webapp=/doxo/ht path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0
[junit4:junit4] 2> 1454720 T2775 C975 P49662 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 1454726 T2775 C975 P49662 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 1454726 T2775 C975 P49662 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 1454733 T2775 C975 P49662 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty4/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty4/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 1454735 T2775 C975 P49662 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1454737 T2775 C975 P49662 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 1454738 T2775 C975 P49662 oass.SolrIndexSearcher.<init> Opening Searcher@3874f79f main
[junit4:junit4] 2> 1454740 T2774 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3874f79f main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 1454741 T2775 C975 P49662 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty4/index.20130614171149503 [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty4/index.20130614171149503;done=true>>]
[junit4:junit4] 2> 1454741 T2775 C975 P49662 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty4/index.20130614171149503
[junit4:junit4] 2> 1454742 T2775 C975 P49662 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty4/index.20130614171149503
[junit4:junit4] 2> 1454743 T2775 C975 P49662 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 1454744 T2775 C975 P49662 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 1454744 T2775 C975 P49662 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1454744 T2775 C975 P49662 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1454748 T2775 C975 P49662 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 1454751 T2790 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371226306415/collection1
[junit4:junit4] 2> 1454752 T2790 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1454754 T2790 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1454755 T2790 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1454769 T2790 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371226306415/collection1/'
[junit4:junit4] 2> 1454772 T2790 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371226306415/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1454773 T2790 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371226306415/collection1/lib/README' to classloader
[junit4:junit4] 2> 1454928 T2790 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1455063 T2790 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1455068 T2790 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1455077 T2790 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1455605 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1455607 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49662_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49662/doxo/ht"}
[junit4:junit4] 2> 1455626 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1455626 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1455627 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1455627 T2772 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1455628 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1455628 T2789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1455628 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1456340 T2790 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1456362 T2790 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1456368 T2790 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1456434 T2790 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1456444 T2790 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1456462 T2790 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1456466 T2790 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1456468 T2790 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1456468 T2790 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1456474 T2790 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1456474 T2790 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1456475 T2790 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1456476 T2790 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1371226306415/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty5/
[junit4:junit4] 2> 1456476 T2790 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f3c2edc
[junit4:junit4] 2> 1456477 T2790 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1456479 T2790 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty5
[junit4:junit4] 2> 1456480 T2790 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty5/index/
[junit4:junit4] 2> 1456481 T2790 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty5/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1456482 T2790 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty5/index
[junit4:junit4] 2> 1456491 T2790 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty5/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1456492 T2790 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1456499 T2790 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1456499 T2790 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1456500 T2790 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1456502 T2790 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1456503 T2790 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1456503 T2790 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1456507 T2790 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1456508 T2790 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1456509 T2790 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1456539 T2790 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1456561 T2790 oass.SolrIndexSearcher.<init> Opening Searcher@2128c99e main
[junit4:junit4] 2> 1456564 T2790 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1456565 T2790 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1456575 T2793 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2128c99e main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1456580 T2790 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1456580 T2790 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49666/doxo/ht collection:collection1 shard:shard2
[junit4:junit4] 2> 1456589 T2790 oasc.ZkController.register We are http://127.0.0.1:49666/doxo/ht/collection1/ and leader is http://127.0.0.1:49656/doxo/ht/collection1/
[junit4:junit4] 2> 1456590 T2790 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49666/doxo/ht
[junit4:junit4] 2> 1456591 T2790 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 1456591 T2790 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C977 name=collection1 org.apache.solr.core.SolrCore@43bb93ab url=http://127.0.0.1:49666/doxo/ht/collection1 node=127.0.0.1:49666_doxo%2Fht C977_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:49666_doxo%2Fht, base_url=http://127.0.0.1:49666/doxo/ht}
[junit4:junit4] 2> 1456592 T2794 C977 P49666 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 1456594 T2794 C977 P49666 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 1456594 T2794 C977 P49666 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 1456594 T2794 C977 P49666 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1456597 T2790 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1456606 T2794 C977 P49666 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1456608 T2682 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1456609 T2682 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1456609 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1456665 T2732 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 6, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 1457167 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1457171 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"6",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49666_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49666/doxo/ht"}
[junit4:junit4] 2> 1457200 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1457201 T2789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1457201 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1457202 T2772 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1457200 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1457200 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1457201 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1457407 T2682 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1457415 T2682 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49673
[junit4:junit4] 2> 1457419 T2682 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1457421 T2682 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1457422 T2682 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371226311435
[junit4:junit4] 2> 1457423 T2682 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371226311435/solr.xml
[junit4:junit4] 2> 1457423 T2682 oasc.CoreContainer.<init> New CoreContainer 792204036
[junit4:junit4] 2> 1457424 T2682 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371226311435/'
[junit4:junit4] 2> 1457425 T2682 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371226311435/'
[junit4:junit4] 2> 1457668 T2732 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 6, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 1457671 T2732 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=6&state=recovering&nodeName=127.0.0.1:49666_doxo%252Fht&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1006
[junit4:junit4] 2> 1457741 T2682 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1457743 T2682 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1457744 T2682 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1457745 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1457746 T2682 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1457747 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1457749 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1457750 T2682 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1457751 T2682 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1457752 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1457799 T2682 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1457802 T2682 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49646/solr
[junit4:junit4] 2> 1457805 T2682 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1457816 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1457824 T2806 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@652dd899 name:ZooKeeperConnection Watcher:127.0.0.1:49646 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1457826 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1457835 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1457863 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1457871 T2808 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@52cf9554 name:ZooKeeperConnection Watcher:127.0.0.1:49646/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1457871 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1457894 T2682 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1458908 T2682 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49673_doxo%2Fht
[junit4:junit4] 2> 1458911 T2682 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49673_doxo%2Fht
[junit4:junit4] 2> 1458923 T2772 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1458927 T2808 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1458929 T2704 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1458929 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1458929 T2789 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1458930 T2789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1458929 T2756 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1458931 T2740 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1458931 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1458931 T2711 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1458932 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1458929 T2724 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1458932 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1458931 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1458941 T2772 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1458956 T2809 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1458956 T2809 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> C977_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:49666_doxo%2Fht, base_url=http://127.0.0.1:49666/doxo/ht}
[junit4:junit4] 2> 1459675 T2794 C977 P49666 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:49656/doxo/ht/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 1459675 T2794 C977 P49666 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:49666/doxo/ht START replicas=[http://127.0.0.1:49656/doxo/ht/collection1/] nUpdates=100
[junit4:junit4] 2> 1459676 T2794 C977 P49666 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 1459677 T2794 C977 P49666 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 1459677 T2794 C977 P49666 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 1459677 T2794 C977 P49666 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 1459678 T2794 C977 P49666 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 1459680 T2794 C977 P49666 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:49656/doxo/ht/collection1/. core=collection1
[junit4:junit4] 2> 1459680 T2794 C977 P49666 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C978 name=collection1 org.apache.solr.core.SolrCore@3c697e21 url=http://127.0.0.1:49656/doxo/ht/collection1 node=127.0.0.1:49656_doxo%2Fht C978_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:49656_doxo%2Fht, base_url=http://127.0.0.1:49656/doxo/ht, leader=true}
[junit4:junit4] 2> 1459739 T2732 C978 P49656 oasc.SolrCore.execute [collection1] webapp=/doxo/ht path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=2
[junit4:junit4] 2> 1459744 T2735 C978 P49656 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 1459752 T2735 C978 P49656 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty2/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1459754 T2735 C978 P49656 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1459760 T2735 C978 P49656 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty2/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty2/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 1459761 T2735 C978 P49656 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1459764 T2735 C978 P49656 oass.SolrIndexSearcher.<init> Opening Searcher@4a53ede realtime
[junit4:junit4] 2> 1459766 T2735 C978 P49656 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1459767 T2735 C978 P49656 oasup.LogUpdateProcessor.finish [collection1] webapp=/doxo/ht path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 24
[junit4:junit4] 2> 1459769 T2794 C977 P49666 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1459770 T2794 C977 P49666 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 1459780 T2731 C978 P49656 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1459781 T2731 C978 P49656 oasc.SolrCore.execute [collection1] webapp=/doxo/ht path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=3
[junit4:junit4] 2> 1459783 T2794 C977 P49666 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 1459784 T2794 C977 P49666 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 1459784 T2794 C977 P49666 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 1459789 T2733 C978 P49656 oasc.SolrCore.execute [collection1] webapp=/doxo/ht path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=1
[junit4:junit4] 2> 1459792 T2794 C977 P49666 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 1459801 T2794 C977 P49666 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty5/index.20130614171154590
[junit4:junit4] 2> 1459802 T2794 C977 P49666 oash.SnapPuller.fetchLatestIndex Starting download to org.apache.lucene.store.NIOFSDirectory@/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty5/index.20130614171154590 lockFactory=org.apache.lucene.store.NativeFSLockFactory@51ac0fc3 fullCopy=false
[junit4:junit4] 2> 1459807 T2734 C978 P49656 oasc.SolrCore.execute [collection1] webapp=/doxo/ht path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0
[junit4:junit4] 2> 1459811 T2794 C977 P49666 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 1459816 T2794 C977 P49666 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 1459816 T2794 C977 P49666 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 1459826 T2794 C977 P49666 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty5/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty5/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 1459828 T2794 C977 P49666 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1459829 T2794 C977 P49666 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 1459831 T2794 C977 P49666 oass.SolrIndexSearcher.<init> Opening Searcher@747a875a main
[junit4:junit4] 2> 1459834 T2793 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@747a875a main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 1459835 T2794 C977 P49666 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty5/index.20130614171154590 [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty5/index.20130614171154590;done=true>>]
[junit4:junit4] 2> 1459836 T2794 C977 P49666 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty5/index.20130614171154590
[junit4:junit4] 2> 1459836 T2794 C977 P49666 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty5/index.20130614171154590
[junit4:junit4] 2> 1459838 T2794 C977 P49666 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 1459838 T2794 C977 P49666 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 1459838 T2794 C977 P49666 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1459839 T2794 C977 P49666 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1459845 T2794 C977 P49666 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 1460243 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1460246 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"7",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49673_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49673/doxo/ht"}
[junit4:junit4] 2> 1460247 T2705 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4:junit4] 2> 1460247 T2705 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard3
[junit4:junit4] 2> 1460272 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"6",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49666_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49666/doxo/ht"}
[junit4:junit4] 2> 1460297 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1460312 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1460313 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1460314 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1460314 T2789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1460315 T2808 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1460316 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1460317 T2772 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1460961 T2809 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371226311435/collection1
[junit4:junit4] 2> 1460962 T2809 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1460964 T2809 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1460965 T2809 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1460969 T2809 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371226311435/collection1/'
[junit4:junit4] 2> 1460971 T2809 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371226311435/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1460972 T2809 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371226311435/collection1/lib/README' to classloader
[junit4:junit4] 2> 1461132 T2809 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 2> 1461226 T2809 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1461237 T2809 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1461246 T2809 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1462501 T2809 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1462518 T2809 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1462530 T2809 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1462573 T2809 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1462583 T2809 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1462602 T2809 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1462613 T2809 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1462615 T2809 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1462616 T2809 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1462621 T2809 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1462622 T2809 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1462623 T2809 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1462623 T2809 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1371226311435/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty6/
[junit4:junit4] 2> 1462624 T2809 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f3c2edc
[junit4:junit4] 2> 1462624 T2809 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1462626 T2809 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty6
[junit4:junit4] 2> 1462627 T2809 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty6/index/
[junit4:junit4] 2> 1462628 T2809 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty6/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1462629 T2809 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty6/index
[junit4:junit4] 2> 1462639 T2809 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty6/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1462640 T2809 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1462647 T2809 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1462648 T2809 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1462649 T2809 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1462651 T2809 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1462654 T2809 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1462654 T2809 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1462659 T2809 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1462660 T2809 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1462661 T2809 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1462697 T2809 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1462713 T2809 oass.SolrIndexSearcher.<init> Opening Searcher@551eedf0 main
[junit4:junit4] 2> 1462716 T2809 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1462717 T2809 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1462725 T2812 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@551eedf0 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1462730 T2809 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1462731 T2809 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49673/doxo/ht collection:collection1 shard:shard3
[junit4:junit4] 2> 1462745 T2809 oasc.ZkController.register We are http://127.0.0.1:49673/doxo/ht/collection1/ and leader is http://127.0.0.1:49659/doxo/ht/collection1/
[junit4:junit4] 2> 1462746 T2809 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49673/doxo/ht
[junit4:junit4] 2> 1462746 T2809 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 1462747 T2809 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C979 name=collection1 org.apache.solr.core.SolrCore@779d3aef url=http://127.0.0.1:49673/doxo/ht/collection1 node=127.0.0.1:49673_doxo%2Fht C979_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:49673_doxo%2Fht, base_url=http://127.0.0.1:49673/doxo/ht}
[junit4:junit4] 2> 1462748 T2813 C979 P49673 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 1462749 T2813 C979 P49673 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 1462750 T2813 C979 P49673 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 1462750 T2813 C979 P49673 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1462754 T2813 C979 P49673 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1462788 T2809 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1462805 T2682 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1462806 T2682 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1462808 T2750 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 7, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 1462808 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1463345 T2705 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1463351 T2705 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"7",
[junit4:junit4] 2> "numShards":"3",
[junit4:junit4] 2> "shard":"shard3",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49673_doxo%2Fht",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49673/doxo/ht"}
[junit4:junit4] 2> 1463373 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1463373 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1463374 T2772 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1463373 T2808 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1463373 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1463374 T2789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1463374 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1463374 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1463609 T2682 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1463617 T2682 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49680
[junit4:junit4] 2> 1463619 T2682 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1463620 T2682 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1463621 T2682 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371226317633
[junit4:junit4] 2> 1463622 T2682 oasc.CoreContainer$Initializer.initialize looking for solr config file: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371226317633/solr.xml
[junit4:junit4] 2> 1463623 T2682 oasc.CoreContainer.<init> New CoreContainer 189571726
[junit4:junit4] 2> 1463624 T2682 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371226317633/'
[junit4:junit4] 2> 1463625 T2682 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty7-1371226317633/'
[junit4:junit4] 2> 1463845 T2750 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 7, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 1463846 T2750 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=7&state=recovering&nodeName=127.0.0.1:49673_doxo%252Fht&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1038
[junit4:junit4] 2> 1463906 T2682 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1463907 T2682 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1463908 T2682 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1463909 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1463909 T2682 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1463910 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1463911 T2682 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1463912 T2682 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1463913 T2682 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1463914 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1463940 T2682 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1463942 T2682 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49646/solr
[junit4:junit4] 2> 1463943 T2682 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1463951 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1463956 T2825 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c4c5c53 name:ZooKeeperConnection Watcher:127.0.0.1:49646 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1463957 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1463966 T2682 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1463989 T2682 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1463995 T2827 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3126ffa0 name:ZooKeeperConnection Watcher:127.0.0.1:49646/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1463996 T2682 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1464009 T2682 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1465024 T2682 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49680_doxo%2Fht
[junit4:junit4] 2> 1465027 T2682 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49680_doxo%2Fht
[junit4:junit4] 2> 1465039 T2772 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1465048 T2724 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1465048 T2724 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4] 2> 1465052 T2740 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1465053 T2740 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4] 2> 1465052 T2711 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1465054 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4] 2> 1465051 T2756 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1465055 T2756 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4] 2> 1465051 T2704 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1465065 T2704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4] 2> 1465049 T2808 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1465051 T2789 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1465066 T2789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 8)
[junit4:junit4] 2> 1465070 T2827 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1465077 T2772 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1465083 T2808 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4:junit4] 2> 1465093 T2828 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1465094 T2828 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> ASYNC NEW_CORE C980 name=collection1 org.apache.solr.core.SolrCore@779d3aef url=http://127.0.0.1:49673/doxo/ht/collection1 node=127.0.0.1:49673_doxo%2Fht C980_STATE=coll:collection1 core:collection1 props:{shard=shard3, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:49673_doxo%2Fht, base_url=http://127.0.0.1:49673/doxo/ht}
[junit4:junit4] 2> 1465859 T2813 C980 P49673 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:49659/doxo/ht/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 1465859 T2813 C980 P49673 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:49673/doxo/ht START replicas=[http://127.0.0.1:49659/doxo/ht/collection1/] nUpdates=100
[junit4:junit4] 2> 1465860 T2813 C980 P49673 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 1465861 T2813 C980 P49673 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 1465861 T2813 C980 P49673 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 1465862 T2813 C980 P49673 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 1465862 T2813 C980 P49673 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 1465862 T2813 C980 P49673 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:49659/doxo/ht/collection1/. core=collection1
[junit4:junit4] 2> 1465865 T2813 C980 P49673 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C981 name=collection1 org.apache.solr.core.SolrCore@a8dfcb9 url=http://127.0.0.1:49659/doxo/ht/collection1 node=127.0.0.1:49659_doxo%2Fht C981_STATE=coll:collection1 core:collection1 props:{shard=shard3, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:49659_doxo%2Fht, base_url=http://127.0.0.1:49659/doxo/ht, leader=true}
[junit4:junit4] 2> 1465875 T2750 C981 P49659 oasc.SolrCore.execute [collection1] webapp=/doxo/ht path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0
[junit4:junit4] 2> 1465893 T2749 C981 P49659 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 1465900 T2749 C981 P49659 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty3/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1465901 T2749 C981 P49659 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1465906 T2749 C981 P49659 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty3/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty3/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 1465907 T2749 C981 P49659 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1465910 T2749 C981 P49659 oass.SolrIndexSearcher.<init> Opening Searcher@7d40a25b realtime
[junit4:junit4] 2> 1465910 T2749 C981 P49659 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1465911 T2749 C981 P49659 oasup.LogUpdateProcessor.finish [collection1] webapp=/doxo/ht path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 18
[junit4:junit4] 2> 1465913 T2813 C980 P49673 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1465913 T2813 C980 P49673 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 1465929 T2752 C981 P49659 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1465930 T2752 C981 P49659 oasc.SolrCore.execute [collection1] webapp=/doxo/ht path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=2
[junit4:junit4] 2> 1465931 T2813 C980 P49673 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 1465931 T2813 C980 P49673 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 1465932 T2813 C980 P49673 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 1465939 T2747 C981 P49659 oasc.SolrCore.execute [collection1] webapp=/doxo/ht path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=2
[junit4:junit4] 2> 1465941 T2813 C980 P49673 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 1465944 T2813 C980 P49673 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty6/index.20130614171200737
[junit4:junit4] 2> 1465945 T2813 C980 P49673 oash.SnapPuller.fetchLatestIndex Starting download to org.apache.lucene.store.NIOFSDirectory@/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty6/index.20130614171200737 lockFactory=org.apache.lucene.store.NativeFSLockFactory@292f96da fullCopy=false
[junit4:junit4] 2> 1465950 T2750 C981 P4
[...truncated too long message...]
rocess WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 1546829 T2682 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1546833 T2682 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1546839 T2682 oasc.SolrCore.close [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@22dbe117
[junit4:junit4] 2> 1546842 T2682 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=26,cumulative_deletesById=5,cumulative_deletesByQuery=1,cumulative_errors=0}
[junit4:junit4] 2> 1546844 T2682 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
[junit4:junit4] 2> 1546845 T2682 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
[junit4:junit4] 2> 1546846 T2682 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
[junit4:junit4] 2> 1546853 T2682 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
[junit4:junit4] 2> 1546856 T2682 oasc.CachingDirectoryFactory.close Closing StandardDirectoryFactory - 2 directories currently being tracked
[junit4:junit4] 2> 1546857 T2682 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty11/index [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty11/index;done=false>>]
[junit4:junit4] 2> 1546858 T2682 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty11/index
[junit4:junit4] 2> 1546859 T2682 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty11 [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty11;done=false>>]
[junit4:junit4] 2> 1546861 T2682 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1371226275504/jetty11
[junit4:junit4] 2> 1546863 T3022 oasc.Overseer$ClusterStateUpdater.amILeader According to ZK I (id=89864685192413210-127.0.0.1:49702_doxo%2Fht-n_0000000011) am no longer a leader.
[junit4:junit4] 2> 1546870 T2903 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1546871 T2903 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 1546872 T2903 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 1546875 T2711 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4:junit4] 2> 1546875 T2711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 0)
[junit4:junit4] 2> 1546886 T2682 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/doxo/ht,null}
[junit4:junit4] 2> 1546943 T2682 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 49707
[junit4:junit4] 2> 1546949 T2682 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 49649
[junit4:junit4] 2> 1547020 T2884 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 1547021 T2884 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 1547180 T2682 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
[junit4:junit4] 2> 1547189 T2682 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:49646 49646
[junit4:junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4:junit4] 1> safe stop:
[junit4:junit4] 1> safe stop:
[junit4:junit4] 2> 1547300 T2682 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4:junit4] 2> 1668519 T2682 oas.SolrTestCaseJ4.endTrackingSearchers ERROR ERROR: SolrIndexSearcher opens=44 closes=43
[junit4:junit4] 2> NOTE: test params are: codec=Lucene40, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=en_IE, timezone=Etc/GMT-1
[junit4:junit4] 2> NOTE: Mac OS X 10.8.4 x86_64/Oracle Corporation 1.7.0_21 (64-bit)/cpus=2,threads=1,free=298892896,total=536870912
[junit4:junit4] 2> NOTE: All tests run in this JVM: [ConvertedLegacyTest, SortByFunctionTest, TestDistributedSearch, DirectUpdateHandlerTest, SuggesterWFSTTest, NotRequiredUniqueKeyTest, ClusterStateUpdateTest, OutputWriterTest, TestSolrQueryParserDefaultOperatorResource, AutoCommitTest, SpellCheckCollatorTest, SolrIndexSplitterTest, BadComponentTest, TestValueSourceCache, PrimitiveFieldTypeTest, SOLR749Test, IndexSchemaTest, TestJmxMonitoredMap, TestDocSet, OpenCloseCoreStressTest, XmlUpdateRequestHandlerTest, ShowFileRequestHandlerTest, TestDistributedGrouping, ReturnFieldsTest, SimplePostToolTest, BadCopyFieldTest, MinimalSchemaTest, JSONWriterTest, TestArbitraryIndexDir, TestAnalyzedSuggestions, TestPseudoReturnFields, TestReplicationHandler, BinaryUpdateRequestHandlerTest, TestOmitPositions, MBeansHandlerTest, UpdateParamsTest, ScriptEngineTest, TestSolrQueryParser, HardAutoCommitTest, TestJoin, TestQuerySenderListener, CurrencyFieldOpenExchangeTest, TestLFUCache, TestGroupingSearch, PluginInfoTest, TestFieldCollectionResource, WordBreakSolrSpellCheckerTest, TestUtils, TestRecovery, LegacyHTMLStripCharFilterTest, TestFaceting, TestFastWriter, TestSolrDeletionPolicy2, TestCloudManagedSchema, TestFieldTypeCollectionResource, LukeRequestHandlerTest, DistributedQueryElevationComponentTest, FieldMutatingUpdateProcessorTest, TestFieldResource, PolyFieldTest, TestRandomDVFaceting, PathHierarchyTokenizerFactoryTest, DocumentBuilderTest, SolrRequestParserTest, TestStressVersions, TestCloudManagedSchemaAddField, TestCharFilters, SolrTestCaseJ4Test, SolrCmdDistributorTest, HighlighterTest, IndexReaderFactoryTest, StatelessScriptUpdateProcessorFactoryTest, DocValuesTest, ShardRoutingCustomTest, TestPropInject, TestPerFieldSimilarity, TestCodecSupport, TestCoreDiscovery, QueryEqualityTest, ZkControllerTest, TestSerializedLuceneMatchVersion, MoreLikeThisHandlerTest, DistanceFunctionTest, TestFieldTypeResource, HighlighterConfigTest, SynonymTokenizerTest, PeerSyncTest, TestQueryUtils, TestRandomFaceting, CoreContainerCoreInitFailuresTest, BasicFunctionalityTest, TestPluginEnable, TestUpdate, ZkSolrClientTest, TestLMDirichletSimilarityFactory, TestJmxIntegration, TestSolrXMLSerializer, TestCSVLoader, TestSchemaResource, QueryElevationComponentTest, TestSolrDeletionPolicy1, TestHashPartitioner, TermVectorComponentTest, IndexSchemaRuntimeFieldTest, TestSearchPerf, TestSolrQueryParserResource, SpellPossibilityIteratorTest, ZkNodePropsTest, SpellingQueryConverterTest, TestManagedSchemaFieldResource, SolrCoreCheckLockOnStartupTest, TestFoldingMultitermQuery, TestCSVResponseWriter, TestWordDelimiterFilterFactory, TestLMJelinekMercerSimilarityFactory, TestCopyFieldCollectionResource, ChaosMonkeySafeLeaderTest]
[junit4:junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.seed=B9A2CFAB6B3FFFF4 -Dtests.slow=true -Dtests.locale=en_IE -Dtests.timezone=Etc/GMT-1 -Dtests.file.encoding=ISO-8859-1
[junit4:junit4] ERROR 0.00s | ChaosMonkeySafeLeaderTest (suite) <<<
[junit4:junit4] > Throwable #1: java.lang.AssertionError: ERROR: SolrIndexSearcher opens=44 closes=43
[junit4:junit4] > at __randomizedtesting.SeedInfo.seed([B9A2CFAB6B3FFFF4]:0)
[junit4:junit4] > at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:254)
[junit4:junit4] > at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:102)
[junit4:junit4] > at java.lang.Thread.run(Thread.java:722)
[junit4:junit4] Completed in 247.92s, 1 test, 1 failure <<< FAILURES!
[...truncated 541 lines...]
BUILD FAILED
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:385: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:365: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:39: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:181: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:437: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1240: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:884: There were test failures: 296 suites, 1243 tests, 1 suite-level error, 18 ignored (6 assumptions)
Total time: 121 minutes 33 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure