You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/04/01 23:32:25 UTC
[JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.7.0_17) - Build #
4970 - Failure!
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/4970/
Java: 32bit/jdk1.7.0_17 -server -XX:+UseG1GC
1 tests failed.
REGRESSION: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch
Error Message:
document count mismatch. control=2356 sum(shards)=2357 cloudClient=2357
Stack Trace:
java.lang.AssertionError: document count mismatch. control=2356 sum(shards)=2357 cloudClient=2357
at __randomizedtesting.SeedInfo.seed([B7037FE5FE461C1:8A96B9E628BB01FD]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1052)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:137)
at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:806)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:601)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:722)
Build Log:
[...truncated 9280 lines...]
[junit4:junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.450; org.apache.solr.BaseDistributedSearchTestCase; Setting hostContext system property: /
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.456; org.apache.solr.SolrTestCaseJ4; ###Starting testDistribSearch
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.457; org.apache.solr.cloud.ZkTestServer; STARTING ZK TEST SERVER
[junit4:junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.458; org.apache.solr.cloud.ZkTestServer$ZKServerMain; Starting server
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.459; org.apache.zookeeper.server.ZooKeeperServer; tickTime set to 1000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.459; org.apache.zookeeper.server.ZooKeeperServer; minSessionTimeout set to -1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.459; org.apache.zookeeper.server.ZooKeeperServer; maxSessionTimeout set to -1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.459; org.apache.zookeeper.server.NIOServerCnxnFactory; binding to port 0.0.0.0/0.0.0.0:0
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.460; org.apache.zookeeper.server.persistence.FileTxnSnapLog; Snapshotting: 0x0 to /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/zookeeper/server1/data/version-2/snapshot.0
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.558; org.apache.solr.cloud.ZkTestServer; start zk server on port:58945
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.559; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945 sessionTimeout=10000 watcher=org.apache.solr.common.cloud.ConnectionManager@2c34a7
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.559; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.560; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.560; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.560; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49318
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.561; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49318
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.561; org.apache.zookeeper.server.persistence.FileTxnLog; Creating new log file: log.1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.624; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0000 with negotiated timeout 10000 for client /127.0.0.1:49318
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.624; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0000, negotiated timeout = 10000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.624; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@2c34a7 name:ZooKeeperConnection Watcher:127.0.0.1:58945 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.625; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.625; org.apache.solr.common.cloud.SolrZkClient; makePath: /solr
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.628; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd0000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.628; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd0000 closed
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.629; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49318 which had sessionid 0x13dc77a08cd0000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.628; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.629; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=10000 watcher=org.apache.solr.common.cloud.ConnectionManager@4b8163
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.630; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.630; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.631; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.631; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49319
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.631; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49319
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.632; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0001 with negotiated timeout 10000 for client /127.0.0.1:49319
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.632; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0001, negotiated timeout = 10000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.633; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@4b8163 name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.633; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.633; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.636; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1/shards
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.638; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/control_collection
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.640; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/control_collection/shards
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.642; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.643; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/solrconfig.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.646; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.647; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/schema.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.649; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.649; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/stopwords.txt
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.651; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.652; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/protwords.txt
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.655; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.655; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/currency.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.658; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.658; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/open-exchange-rates.json
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.660; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.660; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.663; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.663; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/old_synonyms.txt
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.665; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.665; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/synonyms.txt
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.667; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd0001
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.668; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd0001 closed
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.668; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49319 which had sessionid 0x13dc77a08cd0001
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.668; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.728; org.eclipse.jetty.server.Server; jetty-8.1.8.v20121106
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.732; org.eclipse.jetty.server.AbstractConnector; Started SelectChannelConnector@127.0.0.1:43765
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.733; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init()
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.733; org.apache.solr.core.SolrResourceLoader; JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.733; org.apache.solr.core.SolrResourceLoader; using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.734; org.apache.solr.core.CoreContainer$Initializer; looking for solr config file: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/solr.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.734; org.apache.solr.core.CoreContainer; New CoreContainer 22426858
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.735; org.apache.solr.core.CoreContainer; Loading CoreContainer using Solr Home: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.735; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.779; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting socketTimeout to: 120000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.779; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting urlScheme to: http://
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.780; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting connTimeout to: 15000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.780; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxConnectionsPerHost to: 20
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.780; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting corePoolSize to: 0
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.781; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maximumPoolSize to: 2147483647
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.781; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxThreadIdleTime to: 5
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.781; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting sizeOfQueue to: -1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.782; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting fairnessPolicy to: false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.782; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.789; org.apache.solr.core.CoreContainer; Registering Log Listener
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.807; org.apache.solr.core.CoreContainer; Zookeeper client=127.0.0.1:58945/solr
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.808; org.apache.solr.cloud.ZkController; zkHost includes chroot
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.808; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945 sessionTimeout=60000 watcher=org.apache.solr.common.cloud.ConnectionManager@1c3b06a
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.809; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.810; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.810; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.811; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49320
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.811; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49320
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.812; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0002 with negotiated timeout 20000 for client /127.0.0.1:49320
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.812; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0002, negotiated timeout = 20000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.812; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@1c3b06a name:ZooKeeperConnection Watcher:127.0.0.1:58945 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.812; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.814; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd0002
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.814; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd0002 closed
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.814; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.814; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49320 which had sessionid 0x13dc77a08cd0002
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.815; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.819; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=30000 watcher=org.apache.solr.common.cloud.ConnectionManager@14f0398
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.820; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.820; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.821; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.821; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49321
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.822; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49321
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.823; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0003 with negotiated timeout 20000 for client /127.0.0.1:49321
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.823; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0003, negotiated timeout = 20000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.823; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@14f0398 name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.823; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.825; org.apache.solr.common.cloud.SolrZkClient; makePath: /overseer/queue
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.827; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:create cxid:0x7 zxid:0x1b txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.828; org.apache.solr.common.cloud.SolrZkClient; makePath: /overseer/collection-queue-work
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.831; org.apache.solr.common.cloud.SolrZkClient; makePath: /live_nodes
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.833; org.apache.solr.cloud.ZkController; Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43765_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.833; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:delete cxid:0x12 zxid:0x1e txntype:-1 reqpath:n/a Error Path:/solr/live_nodes/127.0.0.1:43765_ Error:KeeperErrorCode = NoNode for /solr/live_nodes/127.0.0.1:43765_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.834; org.apache.solr.common.cloud.SolrZkClient; makePath: /live_nodes/127.0.0.1:43765_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.836; org.apache.solr.common.cloud.SolrZkClient; makePath: /overseer_elect/election
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.841; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:delete cxid:0x20 zxid:0x23 txntype:-1 reqpath:n/a Error Path:/solr/overseer_elect/leader Error:KeeperErrorCode = NoNode for /solr/overseer_elect/leader
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.842; org.apache.solr.common.cloud.SolrZkClient; makePath: /overseer_elect/leader
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.844; org.apache.solr.cloud.Overseer; Overseer (id=89446894564868099-127.0.0.1:43765_-n_0000000000) starting
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.844; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:create cxid:0x25 zxid:0x25 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.845; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:create cxid:0x26 zxid:0x26 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.846; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:create cxid:0x28 zxid:0x27 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.847; org.apache.solr.common.cloud.SolrZkClient; makePath: /overseer/queue-work
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.849; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:create cxid:0x2e zxid:0x29 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.851; org.apache.solr.cloud.OverseerCollectionProcessor; Process current queue of collection creations
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.851; org.apache.solr.common.cloud.SolrZkClient; makePath: /clusterstate.json
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.853; org.apache.solr.common.cloud.SolrZkClient; makePath: /aliases.json
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.855; org.apache.solr.common.cloud.ZkStateReader; Updating cluster state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.857; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Starting to work on the main queue
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.859; org.apache.solr.core.CoreContainer; Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.860; org.apache.solr.cloud.ZkController; Check for collection zkNode:control_collection
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.860; org.apache.solr.cloud.ZkController; Collection zkNode exists
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.860; org.apache.solr.cloud.ZkController; Load collection config from:/collections/control_collection
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.861; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/collection1/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.862; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/collection1/lib/README' to classloader
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.862; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/collection1/lib/classes/' to classloader
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.887; org.apache.solr.core.SolrConfig; Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.920; org.apache.solr.core.SolrConfig; Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.922; org.apache.solr.schema.IndexSchema; Reading Solr Schema from schema.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:36.926; org.apache.solr.schema.IndexSchema; Schema name=test
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.301; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.309; org.apache.solr.schema.IndexSchema; default search field in schema is text
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.313; org.apache.solr.schema.IndexSchema; unique key field: id
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.325; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.330; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.333; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 1> WARN - 2013-04-01 21:21:37.334; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
[junit4:junit4] 1> WARN - 2013-04-01 21:21:37.335; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.335; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 1> WARN - 2013-04-01 21:21:37.336; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
[junit4:junit4] 1> WARN - 2013-04-01 21:21:37.336; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.336; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.342; org.apache.solr.core.SolrCore; [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/control/data/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.343; org.apache.solr.core.JmxMonitoredMap; JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f58c60
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.343; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.344; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/control/data
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.344; org.apache.solr.core.SolrCore; New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/control/data/index/
[junit4:junit4] 1> WARN - 2013-04-01 21:21:37.344; org.apache.solr.core.SolrCore; [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/control/data/index' doesn't exist. Creating new index...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.344; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/control/data/index
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.395; org.apache.solr.core.SolrDeletionPolicy; SolrDeletionPolicy.onCommit: commits:num=1
[junit4:junit4] 1> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/control/data/index,segFN=segments_1,generation=1,filenames=[segments_1]
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.395; org.apache.solr.core.SolrDeletionPolicy; newest commit = 1[segments_1]
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.397; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.398; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.399; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.399; org.apache.solr.core.RequestHandlers; created standard: solr.StandardRequestHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.400; org.apache.solr.core.RequestHandlers; created /get: solr.RealTimeGetHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.400; org.apache.solr.core.RequestHandlers; adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.400; org.apache.solr.core.RequestHandlers; created /replication: solr.ReplicationHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.401; org.apache.solr.core.RequestHandlers; created /update: solr.UpdateRequestHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.401; org.apache.solr.core.RequestHandlers; created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.405; org.apache.solr.handler.loader.XMLLoader; xsltCacheLifetimeSeconds=60
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.409; org.apache.solr.search.SolrIndexSearcher; Opening Searcher@1b013d0 main
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.410; org.apache.solr.update.CommitTracker; Hard AutoCommit: disabled
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.410; org.apache.solr.update.CommitTracker; Soft AutoCommit: disabled
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.415; org.apache.solr.core.SolrCore; [collection1] Registered new searcher Searcher@1b013d0 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.416; org.apache.solr.cloud.ZkController; publishing core=collection1 state=down
[junit4:junit4] 1> INFO - 2013-04-01 21:21:37.416; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.364; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.364; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=1 message={
[junit4:junit4] 1> "operation":"state",
[junit4:junit4] 1> "core_node_name":"1",
[junit4:junit4] 1> "numShards":"1",
[junit4:junit4] 1> "shard":null,
[junit4:junit4] 1> "roles":null,
[junit4:junit4] 1> "state":"down",
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "collection":"control_collection",
[junit4:junit4] 1> "node_name":"127.0.0.1:43765_",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:43765"}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.365; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Create collection control_collection with numShards 1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.366; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Assigning new node to shard shard=shard1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.372; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.417; org.apache.solr.core.CoreContainer; registering core: collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.418; org.apache.solr.cloud.ZkController; Register replica - core:collection1 address:http://127.0.0.1:43765 collection:control_collection shard:shard1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.419; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/control_collection/leader_elect/shard1/election
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.427; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:delete cxid:0x83 zxid:0x35 txntype:-1 reqpath:n/a Error Path:/solr/collections/control_collection/leaders Error:KeeperErrorCode = NoNode for /solr/collections/control_collection/leaders
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.428; org.apache.solr.cloud.ShardLeaderElectionContext; Running the leader process.
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.429; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:create cxid:0x84 zxid:0x36 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.431; org.apache.solr.cloud.ShardLeaderElectionContext; Enough replicas found to continue.
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.432; org.apache.solr.cloud.ShardLeaderElectionContext; I may be the new leader - try and sync
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.432; org.apache.solr.cloud.SyncStrategy; Sync replicas to http://127.0.0.1:43765/collection1/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.433; org.apache.solr.cloud.SyncStrategy; Sync Success - now sync replicas to me
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.433; org.apache.solr.cloud.SyncStrategy; http://127.0.0.1:43765/collection1/ has no replicas
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.433; org.apache.solr.cloud.ShardLeaderElectionContext; I am the new leader: http://127.0.0.1:43765/collection1/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.434; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/control_collection/leaders/shard1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:38.438; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:create cxid:0x8f zxid:0x3a txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.881; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.889; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.894; org.apache.solr.cloud.ZkController; We are http://127.0.0.1:43765/collection1/ and leader is http://127.0.0.1:43765/collection1/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.895; org.apache.solr.cloud.ZkController; No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:43765
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.895; org.apache.solr.cloud.ZkController; I am the leader, no recovery necessary
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.895; org.apache.solr.cloud.ZkController; publishing core=collection1 state=active
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.896; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.907; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.911; org.apache.solr.servlet.SolrDispatchFilter; user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.911; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init() done
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.912; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.915; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.916; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=10000 watcher=org.apache.solr.common.cloud.ConnectionManager@ad924
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.917; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.917; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.918; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.918; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49323
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.918; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49323
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.919; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0004 with negotiated timeout 10000 for client /127.0.0.1:49323
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.919; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0004, negotiated timeout = 10000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.919; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@ad924 name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.919; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.924; org.apache.solr.common.cloud.ZkStateReader; Updating cluster state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:39.936; org.apache.solr.cloud.ChaosMonkey; monkey: init - expire sessions:true cause connection loss:true
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.008; org.eclipse.jetty.server.Server; jetty-8.1.8.v20121106
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.010; org.eclipse.jetty.server.AbstractConnector; Started SelectChannelConnector@127.0.0.1:54571
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.011; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init()
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.011; org.apache.solr.core.SolrResourceLoader; JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.012; org.apache.solr.core.SolrResourceLoader; using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.012; org.apache.solr.core.CoreContainer$Initializer; looking for solr config file: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/solr.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.012; org.apache.solr.core.CoreContainer; New CoreContainer 16492721
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.013; org.apache.solr.core.CoreContainer; Loading CoreContainer using Solr Home: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.013; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.033; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting socketTimeout to: 120000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.033; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting urlScheme to: http://
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.034; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting connTimeout to: 15000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.034; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxConnectionsPerHost to: 20
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.035; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting corePoolSize to: 0
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.035; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maximumPoolSize to: 2147483647
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.035; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxThreadIdleTime to: 5
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.036; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting sizeOfQueue to: -1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.036; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting fairnessPolicy to: false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.036; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.043; org.apache.solr.core.CoreContainer; Registering Log Listener
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.053; org.apache.solr.core.CoreContainer; Zookeeper client=127.0.0.1:58945/solr
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.054; org.apache.solr.cloud.ZkController; zkHost includes chroot
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.054; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945 sessionTimeout=60000 watcher=org.apache.solr.common.cloud.ConnectionManager@125f991
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.055; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.056; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.057; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.057; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49324
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.057; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49324
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.058; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0005 with negotiated timeout 20000 for client /127.0.0.1:49324
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.058; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0005, negotiated timeout = 20000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.059; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@125f991 name:ZooKeeperConnection Watcher:127.0.0.1:58945 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.059; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.060; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd0005
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.061; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd0005 closed
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.061; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.061; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.061; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49324 which had sessionid 0x13dc77a08cd0005
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.064; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=30000 watcher=org.apache.solr.common.cloud.ConnectionManager@322b4d
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.065; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.066; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.066; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.066; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49325
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.067; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49325
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.068; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0006 with negotiated timeout 20000 for client /127.0.0.1:49325
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.068; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0006, negotiated timeout = 20000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.069; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@322b4d name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.069; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.070; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0006 type:create cxid:0x1 zxid:0x48 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.071; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0006 type:create cxid:0x3 zxid:0x49 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:40.076; org.apache.solr.common.cloud.ZkStateReader; Updating cluster state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.081; org.apache.solr.cloud.ZkController; Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54571_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.082; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0006 type:delete cxid:0xd zxid:0x4a txntype:-1 reqpath:n/a Error Path:/solr/live_nodes/127.0.0.1:54571_ Error:KeeperErrorCode = NoNode for /solr/live_nodes/127.0.0.1:54571_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.082; org.apache.solr.common.cloud.SolrZkClient; makePath: /live_nodes/127.0.0.1:54571_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.086; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (2)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.086; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (2)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.087; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (2)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.087; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.101; org.apache.solr.core.CoreContainer; Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.102; org.apache.solr.cloud.ZkController; Check for collection zkNode:collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.104; org.apache.solr.cloud.ZkController; Collection zkNode exists
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.105; org.apache.solr.cloud.ZkController; Load collection config from:/collections/collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.116; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/collection1/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.116; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/collection1/lib/README' to classloader
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.117; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/collection1/lib/classes/' to classloader
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.174; org.apache.solr.core.SolrConfig; Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.300; org.apache.solr.core.SolrConfig; Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.302; org.apache.solr.schema.IndexSchema; Reading Solr Schema from schema.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.310; org.apache.solr.schema.IndexSchema; Schema name=test
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.395; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.396; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=1 message={
[junit4:junit4] 1> "operation":"state",
[junit4:junit4] 1> "core_node_name":"1",
[junit4:junit4] 1> "numShards":"1",
[junit4:junit4] 1> "shard":"shard1",
[junit4:junit4] 1> "roles":null,
[junit4:junit4] 1> "state":"active",
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "collection":"control_collection",
[junit4:junit4] 1> "node_name":"127.0.0.1:43765_",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:43765"}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.401; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.401; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.401; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.813; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.819; org.apache.solr.schema.IndexSchema; default search field in schema is text
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.822; org.apache.solr.schema.IndexSchema; unique key field: id
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.835; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.839; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.852; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 1> WARN - 2013-04-01 21:21:41.853; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
[junit4:junit4] 1> WARN - 2013-04-01 21:21:41.854; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.854; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 1> WARN - 2013-04-01 21:21:41.855; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
[junit4:junit4] 1> WARN - 2013-04-01 21:21:41.855; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.856; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.856; org.apache.solr.core.SolrCore; [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty1/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.856; org.apache.solr.core.JmxMonitoredMap; JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f58c60
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.857; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.857; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.857; org.apache.solr.core.SolrCore; New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty1/index/
[junit4:junit4] 1> WARN - 2013-04-01 21:21:41.857; org.apache.solr.core.SolrCore; [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty1/index' doesn't exist. Creating new index...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.858; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty1/index
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.862; org.apache.solr.core.SolrDeletionPolicy; SolrDeletionPolicy.onCommit: commits:num=1
[junit4:junit4] 1> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty1/index,segFN=segments_1,generation=1,filenames=[segments_1]
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.862; org.apache.solr.core.SolrDeletionPolicy; newest commit = 1[segments_1]
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.864; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.865; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.865; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.866; org.apache.solr.core.RequestHandlers; created standard: solr.StandardRequestHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.867; org.apache.solr.core.RequestHandlers; created /get: solr.RealTimeGetHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.867; org.apache.solr.core.RequestHandlers; adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.867; org.apache.solr.core.RequestHandlers; created /replication: solr.ReplicationHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.868; org.apache.solr.core.RequestHandlers; created /update: solr.UpdateRequestHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.869; org.apache.solr.core.RequestHandlers; created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.873; org.apache.solr.handler.loader.XMLLoader; xsltCacheLifetimeSeconds=60
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.877; org.apache.solr.search.SolrIndexSearcher; Opening Searcher@fb5529 main
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.877; org.apache.solr.update.CommitTracker; Hard AutoCommit: disabled
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.878; org.apache.solr.update.CommitTracker; Soft AutoCommit: disabled
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.882; org.apache.solr.core.SolrCore; [collection1] Registered new searcher Searcher@fb5529 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.883; org.apache.solr.cloud.ZkController; publishing core=collection1 state=down
[junit4:junit4] 1> INFO - 2013-04-01 21:21:41.884; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
[junit4:junit4] 1> INFO - 2013-04-01 21:21:42.907; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:42.908; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
[junit4:junit4] 1> "operation":"state",
[junit4:junit4] 1> "core_node_name":"2",
[junit4:junit4] 1> "numShards":"3",
[junit4:junit4] 1> "shard":null,
[junit4:junit4] 1> "roles":null,
[junit4:junit4] 1> "state":"down",
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "collection":"collection1",
[junit4:junit4] 1> "node_name":"127.0.0.1:54571_",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:54571"}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:42.908; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Create collection collection1 with numShards 3
[junit4:junit4] 1> INFO - 2013-04-01 21:21:42.909; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Assigning new node to shard shard=shard1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:42.914; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:42.914; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:42.915; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:43.885; org.apache.solr.core.CoreContainer; registering core: collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:43.885; org.apache.solr.cloud.ZkController; Register replica - core:collection1 address:http://127.0.0.1:54571 collection:collection1 shard:shard1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:43.886; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1/leader_elect/shard1/election
[junit4:junit4] 1> INFO - 2013-04-01 21:21:43.894; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0006 type:delete cxid:0x4b zxid:0x5a txntype:-1 reqpath:n/a Error Path:/solr/collections/collection1/leaders Error:KeeperErrorCode = NoNode for /solr/collections/collection1/leaders
[junit4:junit4] 1> INFO - 2013-04-01 21:21:43.895; org.apache.solr.cloud.ShardLeaderElectionContext; Running the leader process.
[junit4:junit4] 1> INFO - 2013-04-01 21:21:43.896; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0006 type:create cxid:0x4c zxid:0x5b txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:43.900; org.apache.solr.cloud.ShardLeaderElectionContext; Enough replicas found to continue.
[junit4:junit4] 1> INFO - 2013-04-01 21:21:43.900; org.apache.solr.cloud.ShardLeaderElectionContext; I may be the new leader - try and sync
[junit4:junit4] 1> INFO - 2013-04-01 21:21:43.901; org.apache.solr.cloud.SyncStrategy; Sync replicas to http://127.0.0.1:54571/collection1/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:43.901; org.apache.solr.cloud.SyncStrategy; Sync Success - now sync replicas to me
[junit4:junit4] 1> INFO - 2013-04-01 21:21:43.901; org.apache.solr.cloud.SyncStrategy; http://127.0.0.1:54571/collection1/ has no replicas
[junit4:junit4] 1> INFO - 2013-04-01 21:21:43.901; org.apache.solr.cloud.ShardLeaderElectionContext; I am the new leader: http://127.0.0.1:54571/collection1/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:43.902; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1/leaders/shard1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:43.905; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0006 type:create cxid:0x57 zxid:0x5f txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.420; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.435; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.436; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.436; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.460; org.apache.solr.cloud.ZkController; We are http://127.0.0.1:54571/collection1/ and leader is http://127.0.0.1:54571/collection1/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.460; org.apache.solr.cloud.ZkController; No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54571
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.460; org.apache.solr.cloud.ZkController; I am the leader, no recovery necessary
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.461; org.apache.solr.cloud.ZkController; publishing core=collection1 state=active
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.461; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.463; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.464; org.apache.solr.servlet.SolrDispatchFilter; user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.465; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init() done
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.465; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.570; org.eclipse.jetty.server.Server; jetty-8.1.8.v20121106
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.573; org.eclipse.jetty.server.AbstractConnector; Started SelectChannelConnector@127.0.0.1:58204
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.574; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init()
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.575; org.apache.solr.core.SolrResourceLoader; JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.575; org.apache.solr.core.SolrResourceLoader; using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.576; org.apache.solr.core.CoreContainer$Initializer; looking for solr config file: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/solr.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.576; org.apache.solr.core.CoreContainer; New CoreContainer 22798243
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.577; org.apache.solr.core.CoreContainer; Loading CoreContainer using Solr Home: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.577; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.619; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting socketTimeout to: 120000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.620; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting urlScheme to: http://
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.621; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting connTimeout to: 15000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.622; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxConnectionsPerHost to: 20
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.622; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting corePoolSize to: 0
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.623; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maximumPoolSize to: 2147483647
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.623; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxThreadIdleTime to: 5
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.624; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting sizeOfQueue to: -1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.624; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting fairnessPolicy to: false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.625; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.634; org.apache.solr.core.CoreContainer; Registering Log Listener
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.660; org.apache.solr.core.CoreContainer; Zookeeper client=127.0.0.1:58945/solr
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.660; org.apache.solr.cloud.ZkController; zkHost includes chroot
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.661; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945 sessionTimeout=60000 watcher=org.apache.solr.common.cloud.ConnectionManager@11da345
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.662; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.663; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.662; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.663; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49326
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.665; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49326
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.666; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0007 with negotiated timeout 20000 for client /127.0.0.1:49326
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.666; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0007, negotiated timeout = 20000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.666; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@11da345 name:ZooKeeperConnection Watcher:127.0.0.1:58945 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.667; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.668; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd0007
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.668; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd0007 closed
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.669; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.669; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49326 which had sessionid 0x13dc77a08cd0007
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.669; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.673; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=30000 watcher=org.apache.solr.common.cloud.ConnectionManager@9ce19f
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.679; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.680; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49327
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.680; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.681; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.682; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49327
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.684; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0008 with negotiated timeout 20000 for client /127.0.0.1:49327
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.684; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0008, negotiated timeout = 20000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.684; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@9ce19f name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.685; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.686; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0008 type:create cxid:0x1 zxid:0x6c txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.688; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0008 type:create cxid:0x3 zxid:0x6d txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:44.691; org.apache.solr.common.cloud.ZkStateReader; Updating cluster state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.695; org.apache.solr.cloud.ZkController; Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58204_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.696; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0008 type:delete cxid:0xd zxid:0x6e txntype:-1 reqpath:n/a Error Path:/solr/live_nodes/127.0.0.1:58204_ Error:KeeperErrorCode = NoNode for /solr/live_nodes/127.0.0.1:58204_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.697; org.apache.solr.common.cloud.SolrZkClient; makePath: /live_nodes/127.0.0.1:58204_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.699; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.699; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.701; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.701; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.701; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.703; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.703; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.707; org.apache.solr.core.CoreContainer; Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.707; org.apache.solr.cloud.ZkController; Check for collection zkNode:collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.717; org.apache.solr.cloud.ZkController; Collection zkNode exists
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.718; org.apache.solr.cloud.ZkController; Load collection config from:/collections/collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.720; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/collection1/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.720; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/collection1/lib/README' to classloader
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.720; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/collection1/lib/classes/' to classloader
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.757; org.apache.solr.core.SolrConfig; Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.796; org.apache.solr.core.SolrConfig; Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.797; org.apache.solr.schema.IndexSchema; Reading Solr Schema from schema.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.803; org.apache.solr.schema.IndexSchema; Schema name=test
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.942; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.943; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
[junit4:junit4] 1> "operation":"state",
[junit4:junit4] 1> "core_node_name":"2",
[junit4:junit4] 1> "numShards":"3",
[junit4:junit4] 1> "shard":"shard1",
[junit4:junit4] 1> "roles":null,
[junit4:junit4] 1> "state":"active",
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "collection":"collection1",
[junit4:junit4] 1> "node_name":"127.0.0.1:54571_",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:54571"}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.963; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.963; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.964; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:45.963; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.361; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.368; org.apache.solr.schema.IndexSchema; default search field in schema is text
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.371; org.apache.solr.schema.IndexSchema; unique key field: id
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.390; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.393; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.397; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 1> WARN - 2013-04-01 21:21:46.398; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
[junit4:junit4] 1> WARN - 2013-04-01 21:21:46.399; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.400; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 1> WARN - 2013-04-01 21:21:46.401; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
[junit4:junit4] 1> WARN - 2013-04-01 21:21:46.401; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.402; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.402; org.apache.solr.core.SolrCore; [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty2/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.402; org.apache.solr.core.JmxMonitoredMap; JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f58c60
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.403; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.403; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty2
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.403; org.apache.solr.core.SolrCore; New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty2/index/
[junit4:junit4] 1> WARN - 2013-04-01 21:21:46.404; org.apache.solr.core.SolrCore; [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty2/index' doesn't exist. Creating new index...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.404; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty2/index
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.407; org.apache.solr.core.SolrDeletionPolicy; SolrDeletionPolicy.onCommit: commits:num=1
[junit4:junit4] 1> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty2/index,segFN=segments_1,generation=1,filenames=[segments_1]
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.408; org.apache.solr.core.SolrDeletionPolicy; newest commit = 1[segments_1]
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.410; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.410; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.411; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.411; org.apache.solr.core.RequestHandlers; created standard: solr.StandardRequestHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.412; org.apache.solr.core.RequestHandlers; created /get: solr.RealTimeGetHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.412; org.apache.solr.core.RequestHandlers; adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.412; org.apache.solr.core.RequestHandlers; created /replication: solr.ReplicationHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.412; org.apache.solr.core.RequestHandlers; created /update: solr.UpdateRequestHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.413; org.apache.solr.core.RequestHandlers; created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.416; org.apache.solr.handler.loader.XMLLoader; xsltCacheLifetimeSeconds=60
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.419; org.apache.solr.search.SolrIndexSearcher; Opening Searcher@104284e main
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.420; org.apache.solr.update.CommitTracker; Hard AutoCommit: disabled
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.420; org.apache.solr.update.CommitTracker; Soft AutoCommit: disabled
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.425; org.apache.solr.core.SolrCore; [collection1] Registered new searcher Searcher@104284e main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.427; org.apache.solr.cloud.ZkController; publishing core=collection1 state=down
[junit4:junit4] 1> INFO - 2013-04-01 21:21:46.427; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
[junit4:junit4] 1> INFO - 2013-04-01 21:21:47.468; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:47.469; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
[junit4:junit4] 1> "operation":"state",
[junit4:junit4] 1> "core_node_name":"3",
[junit4:junit4] 1> "numShards":"3",
[junit4:junit4] 1> "shard":null,
[junit4:junit4] 1> "roles":null,
[junit4:junit4] 1> "state":"down",
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "collection":"collection1",
[junit4:junit4] 1> "node_name":"127.0.0.1:58204_",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:58204"}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:47.470; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Collection already exists with numShards=3
[junit4:junit4] 1> INFO - 2013-04-01 21:21:47.470; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Assigning new node to shard shard=shard2
[junit4:junit4] 1> INFO - 2013-04-01 21:21:47.475; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:47.475; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:47.475; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:47.475; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.430; org.apache.solr.core.CoreContainer; registering core: collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.431; org.apache.solr.cloud.ZkController; Register replica - core:collection1 address:http://127.0.0.1:58204 collection:collection1 shard:shard2
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.432; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1/leader_elect/shard2/election
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.441; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0008 type:delete cxid:0x4a zxid:0x7d txntype:-1 reqpath:n/a Error Path:/solr/collections/collection1/leaders/shard2 Error:KeeperErrorCode = NoNode for /solr/collections/collection1/leaders/shard2
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.442; org.apache.solr.cloud.ShardLeaderElectionContext; Running the leader process.
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.445; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0008 type:create cxid:0x4b zxid:0x7e txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.447; org.apache.solr.cloud.ShardLeaderElectionContext; Enough replicas found to continue.
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.448; org.apache.solr.cloud.ShardLeaderElectionContext; I may be the new leader - try and sync
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.448; org.apache.solr.cloud.SyncStrategy; Sync replicas to http://127.0.0.1:58204/collection1/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.448; org.apache.solr.cloud.SyncStrategy; Sync Success - now sync replicas to me
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.448; org.apache.solr.cloud.SyncStrategy; http://127.0.0.1:58204/collection1/ has no replicas
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.449; org.apache.solr.cloud.ShardLeaderElectionContext; I am the new leader: http://127.0.0.1:58204/collection1/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.449; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1/leaders/shard2
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.453; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0008 type:create cxid:0x55 zxid:0x81 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.980; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.988; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.989; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.989; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:48.989; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.006; org.apache.solr.cloud.ZkController; We are http://127.0.0.1:58204/collection1/ and leader is http://127.0.0.1:58204/collection1/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.007; org.apache.solr.cloud.ZkController; No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:58204
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.007; org.apache.solr.cloud.ZkController; I am the leader, no recovery necessary
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.008; org.apache.solr.cloud.ZkController; publishing core=collection1 state=active
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.008; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.010; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.011; org.apache.solr.servlet.SolrDispatchFilter; user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.011; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init() done
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.012; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.109; org.eclipse.jetty.server.Server; jetty-8.1.8.v20121106
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.113; org.eclipse.jetty.server.AbstractConnector; Started SelectChannelConnector@127.0.0.1:46444
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.114; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init()
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.114; org.apache.solr.core.SolrResourceLoader; JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.114; org.apache.solr.core.SolrResourceLoader; using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.115; org.apache.solr.core.CoreContainer$Initializer; looking for solr config file: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/solr.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.115; org.apache.solr.core.CoreContainer; New CoreContainer 22206042
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.116; org.apache.solr.core.CoreContainer; Loading CoreContainer using Solr Home: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.116; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.137; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting socketTimeout to: 120000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.139; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting urlScheme to: http://
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.139; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting connTimeout to: 15000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.140; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxConnectionsPerHost to: 20
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.140; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting corePoolSize to: 0
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.141; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maximumPoolSize to: 2147483647
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.141; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxThreadIdleTime to: 5
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.142; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting sizeOfQueue to: -1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.142; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting fairnessPolicy to: false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.143; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.151; org.apache.solr.core.CoreContainer; Registering Log Listener
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.165; org.apache.solr.core.CoreContainer; Zookeeper client=127.0.0.1:58945/solr
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.165; org.apache.solr.cloud.ZkController; zkHost includes chroot
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.166; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945 sessionTimeout=60000 watcher=org.apache.solr.common.cloud.ConnectionManager@17042f6
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.167; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.177; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.178; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.177; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49328
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.179; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49328
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.188; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0009 with negotiated timeout 20000 for client /127.0.0.1:49328
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.188; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0009, negotiated timeout = 20000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.190; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@17042f6 name:ZooKeeperConnection Watcher:127.0.0.1:58945 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.190; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.191; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd0009
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.192; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd0009 closed
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.192; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.192; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49328 which had sessionid 0x13dc77a08cd0009
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.193; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.199; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=30000 watcher=org.apache.solr.common.cloud.ConnectionManager@549c55
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.200; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.200; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49329
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.200; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.201; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49329
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.202; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd000a with negotiated timeout 20000 for client /127.0.0.1:49329
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.202; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd000a, negotiated timeout = 20000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.204; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.204; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@549c55 name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.204; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.205; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000a type:create cxid:0x1 zxid:0x8e txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.206; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000a type:create cxid:0x3 zxid:0x8f txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:49.208; org.apache.solr.common.cloud.ZkStateReader; Updating cluster state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.211; org.apache.solr.cloud.ZkController; Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46444_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.213; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000a type:delete cxid:0xd zxid:0x90 txntype:-1 reqpath:n/a Error Path:/solr/live_nodes/127.0.0.1:46444_ Error:KeeperErrorCode = NoNode for /solr/live_nodes/127.0.0.1:46444_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.214; org.apache.solr.common.cloud.SolrZkClient; makePath: /live_nodes/127.0.0.1:46444_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.216; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.217; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.217; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.218; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.218; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.219; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.220; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.220; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.227; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.231; org.apache.solr.core.CoreContainer; Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.232; org.apache.solr.cloud.ZkController; Check for collection zkNode:collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.232; org.apache.solr.cloud.ZkController; Collection zkNode exists
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.232; org.apache.solr.cloud.ZkController; Load collection config from:/collections/collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.233; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/collection1/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.234; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/collection1/lib/README' to classloader
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.234; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/collection1/lib/classes/' to classloader
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.261; org.apache.solr.core.SolrConfig; Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.301; org.apache.solr.core.SolrConfig; Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.303; org.apache.solr.schema.IndexSchema; Reading Solr Schema from schema.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.308; org.apache.solr.schema.IndexSchema; Schema name=test
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.497; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.498; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
[junit4:junit4] 1> "operation":"state",
[junit4:junit4] 1> "core_node_name":"3",
[junit4:junit4] 1> "numShards":"3",
[junit4:junit4] 1> "shard":"shard2",
[junit4:junit4] 1> "roles":null,
[junit4:junit4] 1> "state":"active",
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "collection":"collection1",
[junit4:junit4] 1> "node_name":"127.0.0.1:58204_",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:58204"}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.501; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.502; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.502; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.502; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.503; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.829; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.835; org.apache.solr.schema.IndexSchema; default search field in schema is text
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.837; org.apache.solr.schema.IndexSchema; unique key field: id
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.850; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.857; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.861; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 1> WARN - 2013-04-01 21:21:50.863; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
[junit4:junit4] 1> WARN - 2013-04-01 21:21:50.864; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.864; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 1> WARN - 2013-04-01 21:21:50.865; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
[junit4:junit4] 1> WARN - 2013-04-01 21:21:50.866; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.866; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.866; org.apache.solr.core.SolrCore; [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty3/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.867; org.apache.solr.core.JmxMonitoredMap; JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f58c60
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.867; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.868; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty3
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.868; org.apache.solr.core.SolrCore; New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty3/index/
[junit4:junit4] 1> WARN - 2013-04-01 21:21:50.869; org.apache.solr.core.SolrCore; [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty3/index' doesn't exist. Creating new index...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.869; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty3/index
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.873; org.apache.solr.core.SolrDeletionPolicy; SolrDeletionPolicy.onCommit: commits:num=1
[junit4:junit4] 1> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty3/index,segFN=segments_1,generation=1,filenames=[segments_1]
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.873; org.apache.solr.core.SolrDeletionPolicy; newest commit = 1[segments_1]
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.875; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.875; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.876; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.877; org.apache.solr.core.RequestHandlers; created standard: solr.StandardRequestHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.877; org.apache.solr.core.RequestHandlers; created /get: solr.RealTimeGetHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.878; org.apache.solr.core.RequestHandlers; adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.878; org.apache.solr.core.RequestHandlers; created /replication: solr.ReplicationHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.879; org.apache.solr.core.RequestHandlers; created /update: solr.UpdateRequestHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.879; org.apache.solr.core.RequestHandlers; created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.883; org.apache.solr.handler.loader.XMLLoader; xsltCacheLifetimeSeconds=60
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.892; org.apache.solr.search.SolrIndexSearcher; Opening Searcher@ad0057 main
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.893; org.apache.solr.update.CommitTracker; Hard AutoCommit: disabled
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.893; org.apache.solr.update.CommitTracker; Soft AutoCommit: disabled
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.899; org.apache.solr.cloud.ZkController; publishing core=collection1 state=down
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.899; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
[junit4:junit4] 1> INFO - 2013-04-01 21:21:50.902; org.apache.solr.core.SolrCore; [collection1] Registered new searcher Searcher@ad0057 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.009; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.010; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
[junit4:junit4] 1> "operation":"state",
[junit4:junit4] 1> "core_node_name":"4",
[junit4:junit4] 1> "numShards":"3",
[junit4:junit4] 1> "shard":null,
[junit4:junit4] 1> "roles":null,
[junit4:junit4] 1> "state":"down",
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "collection":"collection1",
[junit4:junit4] 1> "node_name":"127.0.0.1:46444_",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:46444"}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.010; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Collection already exists with numShards=3
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.011; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Assigning new node to shard shard=shard3
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.021; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.021; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.021; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.021; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.021; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.902; org.apache.solr.core.CoreContainer; registering core: collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.903; org.apache.solr.cloud.ZkController; Register replica - core:collection1 address:http://127.0.0.1:46444 collection:collection1 shard:shard3
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.906; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1/leader_elect/shard3/election
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.915; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000a type:delete cxid:0x4a zxid:0x9f txntype:-1 reqpath:n/a Error Path:/solr/collections/collection1/leaders/shard3 Error:KeeperErrorCode = NoNode for /solr/collections/collection1/leaders/shard3
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.916; org.apache.solr.cloud.ShardLeaderElectionContext; Running the leader process.
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.917; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000a type:create cxid:0x4b zxid:0xa0 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.920; org.apache.solr.cloud.ShardLeaderElectionContext; Enough replicas found to continue.
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.920; org.apache.solr.cloud.ShardLeaderElectionContext; I may be the new leader - try and sync
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.920; org.apache.solr.cloud.SyncStrategy; Sync replicas to http://127.0.0.1:46444/collection1/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.920; org.apache.solr.cloud.SyncStrategy; Sync Success - now sync replicas to me
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.921; org.apache.solr.cloud.SyncStrategy; http://127.0.0.1:46444/collection1/ has no replicas
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.921; org.apache.solr.cloud.ShardLeaderElectionContext; I am the new leader: http://127.0.0.1:46444/collection1/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.921; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1/leaders/shard3
[junit4:junit4] 1> INFO - 2013-04-01 21:21:52.934; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000a type:create cxid:0x55 zxid:0xa3 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.527; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.540; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.540; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.540; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.540; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.545; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.547; org.apache.solr.cloud.ZkController; We are http://127.0.0.1:46444/collection1/ and leader is http://127.0.0.1:46444/collection1/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.547; org.apache.solr.cloud.ZkController; No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:46444
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.548; org.apache.solr.cloud.ZkController; I am the leader, no recovery necessary
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.548; org.apache.solr.cloud.ZkController; publishing core=collection1 state=active
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.548; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.549; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.551; org.apache.solr.servlet.SolrDispatchFilter; user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.552; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init() done
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.552; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.649; org.eclipse.jetty.server.Server; jetty-8.1.8.v20121106
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.651; org.eclipse.jetty.server.AbstractConnector; Started SelectChannelConnector@127.0.0.1:44384
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.652; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init()
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.652; org.apache.solr.core.SolrResourceLoader; JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.653; org.apache.solr.core.SolrResourceLoader; using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.653; org.apache.solr.core.CoreContainer$Initializer; looking for solr config file: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/solr.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.654; org.apache.solr.core.CoreContainer; New CoreContainer 29587082
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.655; org.apache.solr.core.CoreContainer; Loading CoreContainer using Solr Home: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.655; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.687; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting socketTimeout to: 120000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.688; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting urlScheme to: http://
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.688; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting connTimeout to: 15000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.689; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxConnectionsPerHost to: 20
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.689; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting corePoolSize to: 0
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.690; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maximumPoolSize to: 2147483647
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.690; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxThreadIdleTime to: 5
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.691; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting sizeOfQueue to: -1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.692; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting fairnessPolicy to: false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.692; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.702; org.apache.solr.core.CoreContainer; Registering Log Listener
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.712; org.apache.solr.core.CoreContainer; Zookeeper client=127.0.0.1:58945/solr
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.712; org.apache.solr.cloud.ZkController; zkHost includes chroot
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.713; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945 sessionTimeout=60000 watcher=org.apache.solr.common.cloud.ConnectionManager@1fe7650
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.715; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.717; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.719; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49330
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.720; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.720; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49330
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.721; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd000b with negotiated timeout 20000 for client /127.0.0.1:49330
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.722; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd000b, negotiated timeout = 20000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.722; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@1fe7650 name:ZooKeeperConnection Watcher:127.0.0.1:58945 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.722; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.732; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd000b
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.732; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49330 which had sessionid 0x13dc77a08cd000b
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.736; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd000b closed
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.737; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.738; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.742; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=30000 watcher=org.apache.solr.common.cloud.ConnectionManager@e6a671
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.744; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.744; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.744; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49331
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.745; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.745; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49331
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.746; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd000c with negotiated timeout 20000 for client /127.0.0.1:49331
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.747; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd000c, negotiated timeout = 20000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.748; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@e6a671 name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.749; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.750; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000c type:create cxid:0x1 zxid:0xb0 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.752; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000c type:create cxid:0x3 zxid:0xb1 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:53.754; org.apache.solr.common.cloud.ZkStateReader; Updating cluster state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.758; org.apache.solr.cloud.ZkController; Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44384_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.760; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000c type:delete cxid:0xd zxid:0xb2 txntype:-1 reqpath:n/a Error Path:/solr/live_nodes/127.0.0.1:44384_ Error:KeeperErrorCode = NoNode for /solr/live_nodes/127.0.0.1:44384_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.761; org.apache.solr.common.cloud.SolrZkClient; makePath: /live_nodes/127.0.0.1:44384_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.768; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.768; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.771; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.772; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.772; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.768; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.769; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.773; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.773; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.775; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.776; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.790; org.apache.solr.core.CoreContainer; Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.791; org.apache.solr.cloud.ZkController; Check for collection zkNode:collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.792; org.apache.solr.cloud.ZkController; Collection zkNode exists
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.792; org.apache.solr.cloud.ZkController; Load collection config from:/collections/collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.794; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/collection1/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.794; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/collection1/lib/README' to classloader
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.794; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/collection1/lib/classes/' to classloader
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.834; org.apache.solr.core.SolrConfig; Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.880; org.apache.solr.core.SolrConfig; Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.882; org.apache.solr.schema.IndexSchema; Reading Solr Schema from schema.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:54.888; org.apache.solr.schema.IndexSchema; Schema name=test
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.049; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.050; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
[junit4:junit4] 1> "operation":"state",
[junit4:junit4] 1> "core_node_name":"4",
[junit4:junit4] 1> "numShards":"3",
[junit4:junit4] 1> "shard":"shard3",
[junit4:junit4] 1> "roles":null,
[junit4:junit4] 1> "state":"active",
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "collection":"collection1",
[junit4:junit4] 1> "node_name":"127.0.0.1:46444_",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:46444"}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.055; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.056; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.055; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.055; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.056; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.055; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.460; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.467; org.apache.solr.schema.IndexSchema; default search field in schema is text
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.470; org.apache.solr.schema.IndexSchema; unique key field: id
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.487; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.491; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.495; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 1> WARN - 2013-04-01 21:21:55.498; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
[junit4:junit4] 1> WARN - 2013-04-01 21:21:55.499; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.499; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 1> WARN - 2013-04-01 21:21:55.500; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
[junit4:junit4] 1> WARN - 2013-04-01 21:21:55.500; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.501; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.501; org.apache.solr.core.SolrCore; [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty4/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.501; org.apache.solr.core.JmxMonitoredMap; JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f58c60
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.502; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.502; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty4
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.502; org.apache.solr.core.SolrCore; New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty4/index/
[junit4:junit4] 1> WARN - 2013-04-01 21:21:55.503; org.apache.solr.core.SolrCore; [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty4/index' doesn't exist. Creating new index...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.503; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty4/index
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.506; org.apache.solr.core.SolrDeletionPolicy; SolrDeletionPolicy.onCommit: commits:num=1
[junit4:junit4] 1> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty4/index,segFN=segments_1,generation=1,filenames=[segments_1]
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.506; org.apache.solr.core.SolrDeletionPolicy; newest commit = 1[segments_1]
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.509; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.509; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.510; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.510; org.apache.solr.core.RequestHandlers; created standard: solr.StandardRequestHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.511; org.apache.solr.core.RequestHandlers; created /get: solr.RealTimeGetHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.511; org.apache.solr.core.RequestHandlers; adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.512; org.apache.solr.core.RequestHandlers; created /replication: solr.ReplicationHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.512; org.apache.solr.core.RequestHandlers; created /update: solr.UpdateRequestHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.513; org.apache.solr.core.RequestHandlers; created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.516; org.apache.solr.handler.loader.XMLLoader; xsltCacheLifetimeSeconds=60
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.519; org.apache.solr.search.SolrIndexSearcher; Opening Searcher@f80443 main
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.520; org.apache.solr.update.CommitTracker; Hard AutoCommit: disabled
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.520; org.apache.solr.update.CommitTracker; Soft AutoCommit: disabled
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.526; org.apache.solr.core.SolrCore; [collection1] Registered new searcher Searcher@f80443 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.528; org.apache.solr.cloud.ZkController; publishing core=collection1 state=down
[junit4:junit4] 1> INFO - 2013-04-01 21:21:55.528; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
[junit4:junit4] 1> INFO - 2013-04-01 21:21:56.561; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:56.562; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
[junit4:junit4] 1> "operation":"state",
[junit4:junit4] 1> "core_node_name":"5",
[junit4:junit4] 1> "numShards":"3",
[junit4:junit4] 1> "shard":null,
[junit4:junit4] 1> "roles":null,
[junit4:junit4] 1> "state":"down",
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "collection":"collection1",
[junit4:junit4] 1> "node_name":"127.0.0.1:44384_",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:44384"}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:56.562; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Collection already exists with numShards=3
[junit4:junit4] 1> INFO - 2013-04-01 21:21:56.562; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Assigning new node to shard shard=shard1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:56.566; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:56.567; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:56.567; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:56.567; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:56.567; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:56.567; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.529; org.apache.solr.core.CoreContainer; registering core: collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.530; org.apache.solr.cloud.ZkController; Register replica - core:collection1 address:http://127.0.0.1:44384 collection:collection1 shard:shard1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.533; org.apache.solr.cloud.ZkController; We are http://127.0.0.1:44384/collection1/ and leader is http://127.0.0.1:54571/collection1/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.533; org.apache.solr.cloud.ZkController; No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:44384
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.534; org.apache.solr.cloud.ZkController; Core needs to recover:collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.534; org.apache.solr.update.DefaultSolrCoreState; Running recovery - first canceling any ongoing recovery
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.534; org.apache.solr.cloud.RecoveryStrategy; Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.535; org.apache.solr.cloud.RecoveryStrategy; ###### startupVersions=[]
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.535; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.535; org.apache.solr.cloud.ZkController; publishing core=collection1 state=recovering
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.535; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.536; org.apache.solr.servlet.SolrDispatchFilter; user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.536; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init() done
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.537; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.584; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.597; org.eclipse.jetty.server.Server; jetty-8.1.8.v20121106
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.599; org.eclipse.jetty.server.AbstractConnector; Started SelectChannelConnector@127.0.0.1:43323
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.600; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init()
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.600; org.apache.solr.core.SolrResourceLoader; JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.600; org.apache.solr.core.SolrResourceLoader; using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.601; org.apache.solr.core.CoreContainer$Initializer; looking for solr config file: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/solr.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.601; org.apache.solr.core.CoreContainer; New CoreContainer 13718597
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.601; org.apache.solr.core.CoreContainer; Loading CoreContainer using Solr Home: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.602; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.621; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting socketTimeout to: 120000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.622; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting urlScheme to: http://
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.622; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting connTimeout to: 15000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.623; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxConnectionsPerHost to: 20
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.623; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting corePoolSize to: 0
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.623; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maximumPoolSize to: 2147483647
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.624; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxThreadIdleTime to: 5
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.624; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting sizeOfQueue to: -1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.624; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting fairnessPolicy to: false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.625; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.630; org.apache.solr.core.CoreContainer; Registering Log Listener
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.640; org.apache.solr.core.CoreContainer; Zookeeper client=127.0.0.1:58945/solr
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.641; org.apache.solr.cloud.ZkController; zkHost includes chroot
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.641; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945 sessionTimeout=60000 watcher=org.apache.solr.common.cloud.ConnectionManager@9f2657
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.642; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.642; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.643; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.643; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49334
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.644; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49334
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.645; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd000d with negotiated timeout 20000 for client /127.0.0.1:49334
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.645; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd000d, negotiated timeout = 20000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.645; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@9f2657 name:ZooKeeperConnection Watcher:127.0.0.1:58945 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.646; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.648; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd000d
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.648; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd000d closed
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.648; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.648; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49334 which had sessionid 0x13dc77a08cd000d
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.649; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.652; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=30000 watcher=org.apache.solr.common.cloud.ConnectionManager@ea502d
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.653; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.654; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.654; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.654; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49335
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.655; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49335
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.656; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd000e with negotiated timeout 20000 for client /127.0.0.1:49335
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.656; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd000e, negotiated timeout = 20000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.656; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@ea502d name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.656; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.657; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000e type:create cxid:0x1 zxid:0xc3 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.659; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000e type:create cxid:0x3 zxid:0xc4 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
[junit4:junit4] 1> INFO - 2013-04-01 21:21:57.660; org.apache.solr.common.cloud.ZkStateReader; Updating cluster state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.073; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.074; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
[junit4:junit4] 1> "operation":"state",
[junit4:junit4] 1> "core_node_name":"5",
[junit4:junit4] 1> "numShards":"3",
[junit4:junit4] 1> "shard":"shard1",
[junit4:junit4] 1> "roles":null,
[junit4:junit4] 1> "state":"recovering",
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "collection":"collection1",
[junit4:junit4] 1> "node_name":"127.0.0.1:44384_",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:44384"}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.077; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.077; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.078; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.077; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.077; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.077; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.077; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.589; org.apache.solr.servlet.SolrDispatchFilter; [admin] webapp=null path=/admin/cores params={coreNodeName=5&state=recovering&nodeName=127.0.0.1:44384_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1000
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.663; org.apache.solr.cloud.ZkController; Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43323_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.664; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000e type:delete cxid:0xf zxid:0xc9 txntype:-1 reqpath:n/a Error Path:/solr/live_nodes/127.0.0.1:43323_ Error:KeeperErrorCode = NoNode for /solr/live_nodes/127.0.0.1:43323_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.665; org.apache.solr.common.cloud.SolrZkClient; makePath: /live_nodes/127.0.0.1:43323_
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.667; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.667; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.668; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.667; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.668; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.668; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.668; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.669; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.669; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.669; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.670; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.670; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.670; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.671; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.676; org.apache.solr.core.CoreContainer; Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.676; org.apache.solr.cloud.ZkController; Check for collection zkNode:collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.677; org.apache.solr.cloud.ZkController; Collection zkNode exists
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.677; org.apache.solr.cloud.ZkController; Load collection config from:/collections/collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.678; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/collection1/'
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.678; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/collection1/lib/README' to classloader
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.679; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/collection1/lib/classes/' to classloader
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.702; org.apache.solr.core.SolrConfig; Using Lucene MatchVersion: LUCENE_50
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.731; org.apache.solr.core.SolrConfig; Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.732; org.apache.solr.schema.IndexSchema; Reading Solr Schema from schema.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:58.736; org.apache.solr.schema.IndexSchema; Schema name=test
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.092; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.097; org.apache.solr.schema.IndexSchema; default search field in schema is text
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.100; org.apache.solr.schema.IndexSchema; unique key field: id
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.112; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.115; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.118; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 1> WARN - 2013-04-01 21:21:59.119; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
[junit4:junit4] 1> WARN - 2013-04-01 21:21:59.120; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.120; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 1> WARN - 2013-04-01 21:21:59.121; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
[junit4:junit4] 1> WARN - 2013-04-01 21:21:59.121; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.121; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.122; org.apache.solr.core.SolrCore; [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty5/
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.122; org.apache.solr.core.JmxMonitoredMap; JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f58c60
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.122; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.123; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty5
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.123; org.apache.solr.core.SolrCore; New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty5/index/
[junit4:junit4] 1> WARN - 2013-04-01 21:21:59.123; org.apache.solr.core.SolrCore; [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty5/index' doesn't exist. Creating new index...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.123; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty5/index
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.225; org.apache.solr.core.SolrDeletionPolicy; SolrDeletionPolicy.onCommit: commits:num=1
[junit4:junit4] 1> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty5/index,segFN=segments_1,generation=1,filenames=[segments_1]
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.226; org.apache.solr.core.SolrDeletionPolicy; newest commit = 1[segments_1]
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.227; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.227; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.228; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.228; org.apache.solr.core.RequestHandlers; created standard: solr.StandardRequestHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.229; org.apache.solr.core.RequestHandlers; created /get: solr.RealTimeGetHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.229; org.apache.solr.core.RequestHandlers; adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.229; org.apache.solr.core.RequestHandlers; created /replication: solr.ReplicationHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.230; org.apache.solr.core.RequestHandlers; created /update: solr.UpdateRequestHandler
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.230; org.apache.solr.core.RequestHandlers; created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.233; org.apache.solr.handler.loader.XMLLoader; xsltCacheLifetimeSeconds=60
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.237; org.apache.solr.search.SolrIndexSearcher; Opening Searcher@186d3c7 main
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.238; org.apache.solr.update.CommitTracker; Hard AutoCommit: disabled
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.238; org.apache.solr.update.CommitTracker; Soft AutoCommit: disabled
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.241; org.apache.solr.core.SolrCore; [collection1] Registered new searcher Searcher@186d3c7 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.243; org.apache.solr.cloud.ZkController; publishing core=collection1 state=down
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.243; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.583; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.584; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
[junit4:junit4] 1> "operation":"state",
[junit4:junit4] 1> "core_node_name":"6",
[junit4:junit4] 1> "numShards":"3",
[junit4:junit4] 1> "shard":null,
[junit4:junit4] 1> "roles":null,
[junit4:junit4] 1> "state":"down",
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "collection":"collection1",
[junit4:junit4] 1> "node_name":"127.0.0.1:43323_",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:43323"}
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.584; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Collection already exists with numShards=3
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.584; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Assigning new node to shard shard=shard2
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.588; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.588; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.588; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.588; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.588; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.588; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 1> INFO - 2013-04-01 21:21:59.588; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.244; org.apache.solr.core.CoreContainer; registering core: collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.245; org.apache.solr.cloud.ZkController; Register replica - core:collection1 address:http://127.0.0.1:43323 collection:collection1 shard:shard2
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.247; org.apache.solr.cloud.ZkController; We are http://127.0.0.1:43323/collection1/ and leader is http://127.0.0.1:58204/collection1/
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.247; org.apache.solr.cloud.ZkController; No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:43323
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.247; org.apache.solr.cloud.ZkController; Core needs to recover:collection1
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.248; org.apache.solr.update.DefaultSolrCoreState; Running recovery - first canceling any ongoing recovery
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.248; org.apache.solr.cloud.RecoveryStrategy; Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.249; org.apache.solr.cloud.RecoveryStrategy; ###### startupVersions=[]
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.249; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.249; org.apache.solr.cloud.ZkController; publishing core=collection1 state=recovering
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.249; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.250; org.apache.solr.servlet.SolrDispatchFilter; user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.250; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.250; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init() done
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.251; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.312; org.eclipse.jetty.server.Server; jetty-8.1.8.v20121106
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.314; org.eclipse.jetty.server.AbstractConnector; Started SelectChannelConnector@127.0.0.1:53921
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.314; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init()
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.314; org.apache.solr.core.SolrResourceLoader; JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.315; org.apache.solr.core.SolrResourceLoader; using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1364851320255
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.315; org.apache.solr.core.CoreContainer$Initializer; looking for solr config file: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1364851320255/solr.xml
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.316; org.apache.solr.core.CoreContainer; New CoreContainer 22838968
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.316; org.apache.solr.core.CoreContainer; Loading CoreContainer using Solr Home: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1364851320255/'
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.316; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1364851320255/'
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.335; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting socketTimeout to: 120000
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.335; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting urlScheme to: http://
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.335; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting connTimeout to: 15000
[junit4:junit4] 1> INFO - 2013-04-01 21:22:00.336; org.apache.solr.handler.componen
[...truncated too long message...]
jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./solrtest-ChaosMonkeySafeLeaderTest-1364851296457
[junit4:junit4] 2> added docs:2216 with 0 fails deletes:1051
[junit4:junit4] 2> added docs:2270 with 0 fails deletes:1079
[junit4:junit4] 2> check const of shard1
[junit4:junit4] 2> client0
[junit4:junit4] 2> PROPS:2:{"shard":"shard1","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:54571_","base_url":"http://127.0.0.1:54571","leader":"true"}
[junit4:junit4] 2> live:true
[junit4:junit4] 2> num:775
[junit4:junit4] 2>
[junit4:junit4] 2> client1
[junit4:junit4] 2> PROPS:5:{"shard":"shard1","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:44384_","base_url":"http://127.0.0.1:44384"}
[junit4:junit4] 2> live:true
[junit4:junit4] 2> num:775
[junit4:junit4] 2>
[junit4:junit4] 2> client2
[junit4:junit4] 2> PROPS:8:{"shard":"shard1","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:42744_","base_url":"http://127.0.0.1:42744"}
[junit4:junit4] 2> live:true
[junit4:junit4] 2> num:775
[junit4:junit4] 2>
[junit4:junit4] 2> client3
[junit4:junit4] 2> PROPS:11:{"shard":"shard1","state":"recovery_failed","core":"collection1","collection":"collection1","node_name":"127.0.0.1:47960_","base_url":"http://127.0.0.1:47960"}
[junit4:junit4] 2> error contacting client: Server refused connection at: http://127.0.0.1:47960/collection1
[junit4:junit4] 2>
[junit4:junit4] 2> check const of shard2
[junit4:junit4] 2> client0
[junit4:junit4] 2> PROPS:3:{"shard":"shard2","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:58204_","base_url":"http://127.0.0.1:58204","leader":"true"}
[junit4:junit4] 2> live:true
[junit4:junit4] 2> num:785
[junit4:junit4] 2>
[junit4:junit4] 2> client1
[junit4:junit4] 2> PROPS:6:{"shard":"shard2","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:43323_","base_url":"http://127.0.0.1:43323"}
[junit4:junit4] 2> live:true
[junit4:junit4] 2> num:785
[junit4:junit4] 2>
[junit4:junit4] 2> client2
[junit4:junit4] 2> PROPS:9:{"shard":"shard2","state":"recovering","core":"collection1","collection":"collection1","node_name":"127.0.0.1:41132_","base_url":"http://127.0.0.1:41132"}
[junit4:junit4] 2> live:false
[junit4:junit4] 2> num:51
[junit4:junit4] 2>
[junit4:junit4] 2> client3
[junit4:junit4] 2> PROPS:12:{"shard":"shard2","state":"down","core":"collection1","collection":"collection1","node_name":"127.0.0.1:42437_","base_url":"http://127.0.0.1:42437"}
[junit4:junit4] 2> error contacting client: Server refused connection at: http://127.0.0.1:42437/collection1
[junit4:junit4] 2>
[junit4:junit4] 2> check const of shard3
[junit4:junit4] 2> client0
[junit4:junit4] 2> PROPS:4:{"shard":"shard3","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:46444_","base_url":"http://127.0.0.1:46444","leader":"true"}
[junit4:junit4] 2> live:true
[junit4:junit4] 2> num:797
[junit4:junit4] 2>
[junit4:junit4] 2> client1
[junit4:junit4] 2> PROPS:7:{"shard":"shard3","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:53921_","base_url":"http://127.0.0.1:53921"}
[junit4:junit4] 2> live:true
[junit4:junit4] 2> num:797
[junit4:junit4] 2>
[junit4:junit4] 2> client2
[junit4:junit4] 2> PROPS:10:{"shard":"shard3","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:51794_","base_url":"http://127.0.0.1:51794"}
[junit4:junit4] 2> live:true
[junit4:junit4] 2> num:797
[junit4:junit4] 2>
[junit4:junit4] 2> client3
[junit4:junit4] 2> PROPS:13:{"shard":"shard3","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:42485_","base_url":"http://127.0.0.1:42485"}
[junit4:junit4] 2> live:true
[junit4:junit4] 2> num:797
[junit4:junit4] 2>
[junit4:junit4] 2> http://127.0.0.1:54571/collection1/ : 775
[junit4:junit4] 2> shard:shard1
[junit4:junit4] 2> http://127.0.0.1:58204/collection1/ : 785
[junit4:junit4] 2> shard:shard2
[junit4:junit4] 2> http://127.0.0.1:46444/collection1/ : 797
[junit4:junit4] 2> shard:shard3
[junit4:junit4] 2> ######controlDocList: SolrDocumentList[sz=2356]=[SolrDocument{id=60002}, SolrDocument{id=10002}, SolrDocument{id=60003}, SolrDocument{id=10003}, SolrDocument{id=60007}] , [...] , [SolrDocument{id=12214}, SolrDocument{id=62269}, SolrDocument{id=12215}, SolrDocument{id=62270}, SolrDocument{id=12216}]
[junit4:junit4] 2> ######cloudDocList: SolrDocumentList[sz=2357]=[SolrDocument{id=60002}, SolrDocument{id=10002}, SolrDocument{id=10007}, SolrDocument{id=60007}, SolrDocument{id=60008}] , [...] , [SolrDocument{id=12209}, SolrDocument{id=12210}, SolrDocument{id=12211}, SolrDocument{id=62269}, SolrDocument{id=12215}]
[junit4:junit4] 2> ###### sizes=2356,2357
[junit4:junit4] 2> ###### Only in cloudDocList: [{id=60140}]
[junit4:junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.method=testDistribSearch -Dtests.seed=B7037FE5FE461C1 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar_AE -Dtests.timezone=Etc/GMT-4 -Dtests.file.encoding=US-ASCII
[junit4:junit4] FAILURE 277s J1 | ChaosMonkeySafeLeaderTest.testDistribSearch <<<
[junit4:junit4] > Throwable #1: java.lang.AssertionError: document count mismatch. control=2356 sum(shards)=2357 cloudClient=2357
[junit4:junit4] > at __randomizedtesting.SeedInfo.seed([B7037FE5FE461C1:8A96B9E628BB01FD]:0)
[junit4:junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1052)
[junit4:junit4] > at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:137)
[junit4:junit4] > at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:806)
[junit4:junit4] > at java.lang.Thread.run(Thread.java:722)
[junit4:junit4] 1> INFO - 2013-04-01 21:26:13.794; org.apache.solr.SolrTestCaseJ4; ###deleteCore
[junit4:junit4] 2> 277347 T1680 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 2 leaked thread(s).
[junit4:junit4] 1> INFO - 2013-04-01 21:26:14.001; org.apache.zookeeper.server.SessionTrackerImpl; SessionTrackerImpl exited loop!
[junit4:junit4] 1> WARN - 2013-04-01 21:26:14.963; org.apache.solr.common.cloud.ZkStateReader$3; ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 1> INFO - 2013-04-01 21:26:14.963; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 1> WARN - 2013-04-01 21:26:14.963; org.apache.solr.common.cloud.ZkStateReader$2; ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 1> INFO - 2013-04-01 21:26:14.964; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
[junit4:junit4] 2> NOTE: test params are: codec=SimpleText, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=ar_AE, timezone=Etc/GMT-4
[junit4:junit4] 2> NOTE: Linux 3.2.0-39-generic i386/Oracle Corporation 1.7.0_17 (32-bit)/cpus=8,threads=2,free=88868056,total=329252864
[junit4:junit4] 2> NOTE: All tests run in this JVM: [ZkSolrClientTest, SolrCmdDistributorTest, TestMergePolicyConfig, ZkNodePropsTest, TestBinaryResponseWriter, EchoParamsTest, TestSuggestSpellingConverter, TestJmxIntegration, RequiredFieldsTest, UniqFieldsUpdateProcessorFactoryTest, SolrCoreCheckLockOnStartupTest, DOMUtilTest, TestNumberUtils, TestSolrDiscoveryProperties, TestSchemaNameResource, TestIndexSearcher, TestZkChroot, SpellCheckCollatorTest, SimplePostToolTest, TestMultiCoreConfBootstrap, XmlUpdateRequestHandlerTest, TestSolrQueryParserDefaultOperatorResource, TestRTGBase, TestBM25SimilarityFactory, TestUpdate, BadCopyFieldTest, TestCollationField, TestStressLucene, NoCacheHeaderTest, PathHierarchyTokenizerFactoryTest, AliasIntegrationTest, TestCloudManagedSchema, TestUniqueKeyFieldResource, SolrRequestParserTest, TestFiltering, DistanceFunctionTest, IndexBasedSpellCheckerTest, UpdateParamsTest, FieldAnalysisRequestHandlerTest, TestAnalyzedSuggestions, RequestHandlersTest, OpenCloseCoreStressTest, TestBinaryField, ShardRoutingCustomTest, TestCSVLoader, TestQuerySenderNoQuery, TestIndexingPerformance, HighlighterTest, SpellCheckComponentTest, BasicDistributedZk2Test, ChaosMonkeySafeLeaderTest]
[junit4:junit4] Completed on J1 in 278.62s, 1 test, 1 failure <<< FAILURES!
[...truncated 488 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:375: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:355: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:183: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:448: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1231: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:875: There were test failures: 284 suites, 1173 tests, 1 failure, 13 ignored (7 assumptions)
Total time: 46 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Description set: Java: 32bit/jdk1.7.0_17 -server -XX:+UseG1GC
Email was triggered for: Failure
Sending email for trigger: Failure
Re: [JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.7.0_17) - Build # 4970 - Failure!
Posted by Mark Miller <ma...@gmail.com>.
Looks like a test timeout too low issue as Yonik tracked down in the past. If it keeps up, we should raise the test so/conn timeouts probably.
- Mark
On Apr 1, 2013, at 5:32 PM, Policeman Jenkins Server <je...@thetaphi.de> wrote:
> Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/4970/
> Java: 32bit/jdk1.7.0_17 -server -XX:+UseG1GC
>
> 1 tests failed.
> REGRESSION: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch
>
> Error Message:
> document count mismatch. control=2356 sum(shards)=2357 cloudClient=2357
>
> Stack Trace:
> java.lang.AssertionError: document count mismatch. control=2356 sum(shards)=2357 cloudClient=2357
> at __randomizedtesting.SeedInfo.seed([B7037FE5FE461C1:8A96B9E628BB01FD]:0)
> at org.junit.Assert.fail(Assert.java:93)
> at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1052)
> at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:137)
> at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:806)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:601)
> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
> at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> at java.lang.Thread.run(Thread.java:722)
>
>
>
>
> Build Log:
> [...truncated 9280 lines...]
> [junit4:junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.450; org.apache.solr.BaseDistributedSearchTestCase; Setting hostContext system property: /
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.456; org.apache.solr.SolrTestCaseJ4; ###Starting testDistribSearch
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.457; org.apache.solr.cloud.ZkTestServer; STARTING ZK TEST SERVER
> [junit4:junit4] 1> client port:0.0.0.0/0.0.0.0:0
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.458; org.apache.solr.cloud.ZkTestServer$ZKServerMain; Starting server
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.459; org.apache.zookeeper.server.ZooKeeperServer; tickTime set to 1000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.459; org.apache.zookeeper.server.ZooKeeperServer; minSessionTimeout set to -1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.459; org.apache.zookeeper.server.ZooKeeperServer; maxSessionTimeout set to -1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.459; org.apache.zookeeper.server.NIOServerCnxnFactory; binding to port 0.0.0.0/0.0.0.0:0
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.460; org.apache.zookeeper.server.persistence.FileTxnSnapLog; Snapshotting: 0x0 to /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/zookeeper/server1/data/version-2/snapshot.0
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.558; org.apache.solr.cloud.ZkTestServer; start zk server on port:58945
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.559; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945 sessionTimeout=10000 watcher=org.apache.solr.common.cloud.ConnectionManager@2c34a7
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.559; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.560; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.560; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.560; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49318
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.561; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49318
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.561; org.apache.zookeeper.server.persistence.FileTxnLog; Creating new log file: log.1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.624; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0000 with negotiated timeout 10000 for client /127.0.0.1:49318
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.624; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0000, negotiated timeout = 10000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.624; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@2c34a7 name:ZooKeeperConnection Watcher:127.0.0.1:58945 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.625; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.625; org.apache.solr.common.cloud.SolrZkClient; makePath: /solr
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.628; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd0000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.628; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd0000 closed
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.629; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49318 which had sessionid 0x13dc77a08cd0000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.628; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.629; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=10000 watcher=org.apache.solr.common.cloud.ConnectionManager@4b8163
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.630; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.630; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.631; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.631; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49319
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.631; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49319
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.632; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0001 with negotiated timeout 10000 for client /127.0.0.1:49319
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.632; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0001, negotiated timeout = 10000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.633; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@4b8163 name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.633; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.633; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.636; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1/shards
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.638; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/control_collection
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.640; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/control_collection/shards
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.642; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.643; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/solrconfig.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.646; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.647; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/schema.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.649; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.649; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/stopwords.txt
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.651; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.652; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/protwords.txt
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.655; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.655; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/currency.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.658; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.658; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/open-exchange-rates.json
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.660; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.660; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.663; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.663; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/old_synonyms.txt
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.665; org.apache.solr.cloud.AbstractZkTestCase; put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.665; org.apache.solr.common.cloud.SolrZkClient; makePath: /configs/conf1/synonyms.txt
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.667; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd0001
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.668; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd0001 closed
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.668; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49319 which had sessionid 0x13dc77a08cd0001
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.668; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.728; org.eclipse.jetty.server.Server; jetty-8.1.8.v20121106
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.732; org.eclipse.jetty.server.AbstractConnector; Started SelectChannelConnector@127.0.0.1:43765
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.733; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init()
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.733; org.apache.solr.core.SolrResourceLoader; JNDI not configured for solr (NoInitialContextEx)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.733; org.apache.solr.core.SolrResourceLoader; using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.734; org.apache.solr.core.CoreContainer$Initializer; looking for solr config file: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/solr.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.734; org.apache.solr.core.CoreContainer; New CoreContainer 22426858
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.735; org.apache.solr.core.CoreContainer; Loading CoreContainer using Solr Home: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.735; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.779; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting socketTimeout to: 120000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.779; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting urlScheme to: http://
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.780; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting connTimeout to: 15000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.780; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxConnectionsPerHost to: 20
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.780; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting corePoolSize to: 0
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.781; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maximumPoolSize to: 2147483647
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.781; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxThreadIdleTime to: 5
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.781; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting sizeOfQueue to: -1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.782; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting fairnessPolicy to: false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.782; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.789; org.apache.solr.core.CoreContainer; Registering Log Listener
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.807; org.apache.solr.core.CoreContainer; Zookeeper client=127.0.0.1:58945/solr
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.808; org.apache.solr.cloud.ZkController; zkHost includes chroot
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.808; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945 sessionTimeout=60000 watcher=org.apache.solr.common.cloud.ConnectionManager@1c3b06a
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.809; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.810; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.810; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.811; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49320
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.811; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49320
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.812; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0002 with negotiated timeout 20000 for client /127.0.0.1:49320
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.812; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0002, negotiated timeout = 20000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.812; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@1c3b06a name:ZooKeeperConnection Watcher:127.0.0.1:58945 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.812; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.814; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd0002
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.814; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd0002 closed
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.814; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.814; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49320 which had sessionid 0x13dc77a08cd0002
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.815; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.819; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=30000 watcher=org.apache.solr.common.cloud.ConnectionManager@14f0398
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.820; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.820; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.821; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.821; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49321
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.822; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49321
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.823; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0003 with negotiated timeout 20000 for client /127.0.0.1:49321
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.823; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0003, negotiated timeout = 20000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.823; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@14f0398 name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.823; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.825; org.apache.solr.common.cloud.SolrZkClient; makePath: /overseer/queue
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.827; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:create cxid:0x7 zxid:0x1b txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.828; org.apache.solr.common.cloud.SolrZkClient; makePath: /overseer/collection-queue-work
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.831; org.apache.solr.common.cloud.SolrZkClient; makePath: /live_nodes
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.833; org.apache.solr.cloud.ZkController; Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43765_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.833; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:delete cxid:0x12 zxid:0x1e txntype:-1 reqpath:n/a Error Path:/solr/live_nodes/127.0.0.1:43765_ Error:KeeperErrorCode = NoNode for /solr/live_nodes/127.0.0.1:43765_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.834; org.apache.solr.common.cloud.SolrZkClient; makePath: /live_nodes/127.0.0.1:43765_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.836; org.apache.solr.common.cloud.SolrZkClient; makePath: /overseer_elect/election
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.841; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:delete cxid:0x20 zxid:0x23 txntype:-1 reqpath:n/a Error Path:/solr/overseer_elect/leader Error:KeeperErrorCode = NoNode for /solr/overseer_elect/leader
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.842; org.apache.solr.common.cloud.SolrZkClient; makePath: /overseer_elect/leader
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.844; org.apache.solr.cloud.Overseer; Overseer (id=89446894564868099-127.0.0.1:43765_-n_0000000000) starting
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.844; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:create cxid:0x25 zxid:0x25 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.845; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:create cxid:0x26 zxid:0x26 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.846; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:create cxid:0x28 zxid:0x27 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.847; org.apache.solr.common.cloud.SolrZkClient; makePath: /overseer/queue-work
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.849; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:create cxid:0x2e zxid:0x29 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.851; org.apache.solr.cloud.OverseerCollectionProcessor; Process current queue of collection creations
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.851; org.apache.solr.common.cloud.SolrZkClient; makePath: /clusterstate.json
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.853; org.apache.solr.common.cloud.SolrZkClient; makePath: /aliases.json
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.855; org.apache.solr.common.cloud.ZkStateReader; Updating cluster state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.857; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Starting to work on the main queue
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.859; org.apache.solr.core.CoreContainer; Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.860; org.apache.solr.cloud.ZkController; Check for collection zkNode:control_collection
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.860; org.apache.solr.cloud.ZkController; Collection zkNode exists
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.860; org.apache.solr.cloud.ZkController; Load collection config from:/collections/control_collection
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.861; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/collection1/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.862; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/collection1/lib/README' to classloader
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.862; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/collection1/lib/classes/' to classloader
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.887; org.apache.solr.core.SolrConfig; Using Lucene MatchVersion: LUCENE_50
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.920; org.apache.solr.core.SolrConfig; Loaded SolrConfig: solrconfig.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.922; org.apache.solr.schema.IndexSchema; Reading Solr Schema from schema.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:36.926; org.apache.solr.schema.IndexSchema; Schema name=test
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.301; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.309; org.apache.solr.schema.IndexSchema; default search field in schema is text
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.313; org.apache.solr.schema.IndexSchema; unique key field: id
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.325; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.330; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.333; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:37.334; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:37.335; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.335; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:37.336; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:37.336; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.336; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.342; org.apache.solr.core.SolrCore; [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1364851296669/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/control/data/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.343; org.apache.solr.core.JmxMonitoredMap; JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f58c60
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.343; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.344; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/control/data
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.344; org.apache.solr.core.SolrCore; New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/control/data/index/
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:37.344; org.apache.solr.core.SolrCore; [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/control/data/index' doesn't exist. Creating new index...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.344; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/control/data/index
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.395; org.apache.solr.core.SolrDeletionPolicy; SolrDeletionPolicy.onCommit: commits:num=1
> [junit4:junit4] 1> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/control/data/index,segFN=segments_1,generation=1,filenames=[segments_1]
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.395; org.apache.solr.core.SolrDeletionPolicy; newest commit = 1[segments_1]
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.397; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.398; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.399; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.399; org.apache.solr.core.RequestHandlers; created standard: solr.StandardRequestHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.400; org.apache.solr.core.RequestHandlers; created /get: solr.RealTimeGetHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.400; org.apache.solr.core.RequestHandlers; adding lazy requestHandler: solr.ReplicationHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.400; org.apache.solr.core.RequestHandlers; created /replication: solr.ReplicationHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.401; org.apache.solr.core.RequestHandlers; created /update: solr.UpdateRequestHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.401; org.apache.solr.core.RequestHandlers; created /admin/: org.apache.solr.handler.admin.AdminHandlers
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.405; org.apache.solr.handler.loader.XMLLoader; xsltCacheLifetimeSeconds=60
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.409; org.apache.solr.search.SolrIndexSearcher; Opening Searcher@1b013d0 main
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.410; org.apache.solr.update.CommitTracker; Hard AutoCommit: disabled
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.410; org.apache.solr.update.CommitTracker; Soft AutoCommit: disabled
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.415; org.apache.solr.core.SolrCore; [collection1] Registered new searcher Searcher@1b013d0 main{StandardDirectoryReader(segments_1:1)}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.416; org.apache.solr.cloud.ZkController; publishing core=collection1 state=down
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:37.416; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.364; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.364; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=1 message={
> [junit4:junit4] 1> "operation":"state",
> [junit4:junit4] 1> "core_node_name":"1",
> [junit4:junit4] 1> "numShards":"1",
> [junit4:junit4] 1> "shard":null,
> [junit4:junit4] 1> "roles":null,
> [junit4:junit4] 1> "state":"down",
> [junit4:junit4] 1> "core":"collection1",
> [junit4:junit4] 1> "collection":"control_collection",
> [junit4:junit4] 1> "node_name":"127.0.0.1:43765_",
> [junit4:junit4] 1> "base_url":"http://127.0.0.1:43765"}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.365; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Create collection control_collection with numShards 1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.366; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Assigning new node to shard shard=shard1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.372; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.417; org.apache.solr.core.CoreContainer; registering core: collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.418; org.apache.solr.cloud.ZkController; Register replica - core:collection1 address:http://127.0.0.1:43765 collection:control_collection shard:shard1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.419; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/control_collection/leader_elect/shard1/election
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.427; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:delete cxid:0x83 zxid:0x35 txntype:-1 reqpath:n/a Error Path:/solr/collections/control_collection/leaders Error:KeeperErrorCode = NoNode for /solr/collections/control_collection/leaders
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.428; org.apache.solr.cloud.ShardLeaderElectionContext; Running the leader process.
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.429; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:create cxid:0x84 zxid:0x36 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.431; org.apache.solr.cloud.ShardLeaderElectionContext; Enough replicas found to continue.
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.432; org.apache.solr.cloud.ShardLeaderElectionContext; I may be the new leader - try and sync
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.432; org.apache.solr.cloud.SyncStrategy; Sync replicas to http://127.0.0.1:43765/collection1/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.433; org.apache.solr.cloud.SyncStrategy; Sync Success - now sync replicas to me
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.433; org.apache.solr.cloud.SyncStrategy; http://127.0.0.1:43765/collection1/ has no replicas
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.433; org.apache.solr.cloud.ShardLeaderElectionContext; I am the new leader: http://127.0.0.1:43765/collection1/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.434; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/control_collection/leaders/shard1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:38.438; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0003 type:create cxid:0x8f zxid:0x3a txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.881; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.889; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.894; org.apache.solr.cloud.ZkController; We are http://127.0.0.1:43765/collection1/ and leader is http://127.0.0.1:43765/collection1/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.895; org.apache.solr.cloud.ZkController; No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:43765
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.895; org.apache.solr.cloud.ZkController; I am the leader, no recovery necessary
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.895; org.apache.solr.cloud.ZkController; publishing core=collection1 state=active
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.896; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.907; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.911; org.apache.solr.servlet.SolrDispatchFilter; user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.911; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init() done
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.912; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.915; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.916; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=10000 watcher=org.apache.solr.common.cloud.ConnectionManager@ad924
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.917; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.917; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.918; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.918; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49323
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.918; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49323
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.919; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0004 with negotiated timeout 10000 for client /127.0.0.1:49323
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.919; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0004, negotiated timeout = 10000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.919; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@ad924 name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.919; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.924; org.apache.solr.common.cloud.ZkStateReader; Updating cluster state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:39.936; org.apache.solr.cloud.ChaosMonkey; monkey: init - expire sessions:true cause connection loss:true
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.008; org.eclipse.jetty.server.Server; jetty-8.1.8.v20121106
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.010; org.eclipse.jetty.server.AbstractConnector; Started SelectChannelConnector@127.0.0.1:54571
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.011; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init()
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.011; org.apache.solr.core.SolrResourceLoader; JNDI not configured for solr (NoInitialContextEx)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.012; org.apache.solr.core.SolrResourceLoader; using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.012; org.apache.solr.core.CoreContainer$Initializer; looking for solr config file: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/solr.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.012; org.apache.solr.core.CoreContainer; New CoreContainer 16492721
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.013; org.apache.solr.core.CoreContainer; Loading CoreContainer using Solr Home: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.013; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.033; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting socketTimeout to: 120000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.033; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting urlScheme to: http://
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.034; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting connTimeout to: 15000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.034; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxConnectionsPerHost to: 20
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.035; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting corePoolSize to: 0
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.035; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maximumPoolSize to: 2147483647
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.035; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxThreadIdleTime to: 5
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.036; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting sizeOfQueue to: -1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.036; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting fairnessPolicy to: false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.036; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.043; org.apache.solr.core.CoreContainer; Registering Log Listener
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.053; org.apache.solr.core.CoreContainer; Zookeeper client=127.0.0.1:58945/solr
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.054; org.apache.solr.cloud.ZkController; zkHost includes chroot
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.054; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945 sessionTimeout=60000 watcher=org.apache.solr.common.cloud.ConnectionManager@125f991
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.055; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.056; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.057; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.057; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49324
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.057; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49324
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.058; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0005 with negotiated timeout 20000 for client /127.0.0.1:49324
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.058; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0005, negotiated timeout = 20000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.059; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@125f991 name:ZooKeeperConnection Watcher:127.0.0.1:58945 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.059; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.060; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd0005
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.061; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd0005 closed
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.061; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.061; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.061; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49324 which had sessionid 0x13dc77a08cd0005
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.064; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=30000 watcher=org.apache.solr.common.cloud.ConnectionManager@322b4d
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.065; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.066; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.066; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.066; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49325
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.067; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49325
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.068; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0006 with negotiated timeout 20000 for client /127.0.0.1:49325
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.068; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0006, negotiated timeout = 20000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.069; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@322b4d name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.069; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.070; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0006 type:create cxid:0x1 zxid:0x48 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.071; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0006 type:create cxid:0x3 zxid:0x49 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:40.076; org.apache.solr.common.cloud.ZkStateReader; Updating cluster state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.081; org.apache.solr.cloud.ZkController; Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54571_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.082; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0006 type:delete cxid:0xd zxid:0x4a txntype:-1 reqpath:n/a Error Path:/solr/live_nodes/127.0.0.1:54571_ Error:KeeperErrorCode = NoNode for /solr/live_nodes/127.0.0.1:54571_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.082; org.apache.solr.common.cloud.SolrZkClient; makePath: /live_nodes/127.0.0.1:54571_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.086; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (2)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.086; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (2)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.087; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (2)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.087; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.101; org.apache.solr.core.CoreContainer; Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.102; org.apache.solr.cloud.ZkController; Check for collection zkNode:collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.104; org.apache.solr.cloud.ZkController; Collection zkNode exists
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.105; org.apache.solr.cloud.ZkController; Load collection config from:/collections/collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.116; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/collection1/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.116; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/collection1/lib/README' to classloader
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.117; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/collection1/lib/classes/' to classloader
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.174; org.apache.solr.core.SolrConfig; Using Lucene MatchVersion: LUCENE_50
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.300; org.apache.solr.core.SolrConfig; Loaded SolrConfig: solrconfig.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.302; org.apache.solr.schema.IndexSchema; Reading Solr Schema from schema.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.310; org.apache.solr.schema.IndexSchema; Schema name=test
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.395; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.396; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=1 message={
> [junit4:junit4] 1> "operation":"state",
> [junit4:junit4] 1> "core_node_name":"1",
> [junit4:junit4] 1> "numShards":"1",
> [junit4:junit4] 1> "shard":"shard1",
> [junit4:junit4] 1> "roles":null,
> [junit4:junit4] 1> "state":"active",
> [junit4:junit4] 1> "core":"collection1",
> [junit4:junit4] 1> "collection":"control_collection",
> [junit4:junit4] 1> "node_name":"127.0.0.1:43765_",
> [junit4:junit4] 1> "base_url":"http://127.0.0.1:43765"}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.401; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.401; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.401; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.813; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.819; org.apache.solr.schema.IndexSchema; default search field in schema is text
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.822; org.apache.solr.schema.IndexSchema; unique key field: id
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.835; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.839; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.852; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:41.853; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:41.854; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.854; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:41.855; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:41.855; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.856; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.856; org.apache.solr.core.SolrCore; [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1364851299937/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty1/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.856; org.apache.solr.core.JmxMonitoredMap; JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f58c60
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.857; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.857; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.857; org.apache.solr.core.SolrCore; New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty1/index/
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:41.857; org.apache.solr.core.SolrCore; [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty1/index' doesn't exist. Creating new index...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.858; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty1/index
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.862; org.apache.solr.core.SolrDeletionPolicy; SolrDeletionPolicy.onCommit: commits:num=1
> [junit4:junit4] 1> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty1/index,segFN=segments_1,generation=1,filenames=[segments_1]
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.862; org.apache.solr.core.SolrDeletionPolicy; newest commit = 1[segments_1]
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.864; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.865; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.865; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.866; org.apache.solr.core.RequestHandlers; created standard: solr.StandardRequestHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.867; org.apache.solr.core.RequestHandlers; created /get: solr.RealTimeGetHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.867; org.apache.solr.core.RequestHandlers; adding lazy requestHandler: solr.ReplicationHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.867; org.apache.solr.core.RequestHandlers; created /replication: solr.ReplicationHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.868; org.apache.solr.core.RequestHandlers; created /update: solr.UpdateRequestHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.869; org.apache.solr.core.RequestHandlers; created /admin/: org.apache.solr.handler.admin.AdminHandlers
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.873; org.apache.solr.handler.loader.XMLLoader; xsltCacheLifetimeSeconds=60
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.877; org.apache.solr.search.SolrIndexSearcher; Opening Searcher@fb5529 main
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.877; org.apache.solr.update.CommitTracker; Hard AutoCommit: disabled
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.878; org.apache.solr.update.CommitTracker; Soft AutoCommit: disabled
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.882; org.apache.solr.core.SolrCore; [collection1] Registered new searcher Searcher@fb5529 main{StandardDirectoryReader(segments_1:1)}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.883; org.apache.solr.cloud.ZkController; publishing core=collection1 state=down
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:41.884; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:42.907; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:42.908; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
> [junit4:junit4] 1> "operation":"state",
> [junit4:junit4] 1> "core_node_name":"2",
> [junit4:junit4] 1> "numShards":"3",
> [junit4:junit4] 1> "shard":null,
> [junit4:junit4] 1> "roles":null,
> [junit4:junit4] 1> "state":"down",
> [junit4:junit4] 1> "core":"collection1",
> [junit4:junit4] 1> "collection":"collection1",
> [junit4:junit4] 1> "node_name":"127.0.0.1:54571_",
> [junit4:junit4] 1> "base_url":"http://127.0.0.1:54571"}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:42.908; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Create collection collection1 with numShards 3
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:42.909; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Assigning new node to shard shard=shard1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:42.914; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:42.914; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:42.915; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:43.885; org.apache.solr.core.CoreContainer; registering core: collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:43.885; org.apache.solr.cloud.ZkController; Register replica - core:collection1 address:http://127.0.0.1:54571 collection:collection1 shard:shard1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:43.886; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1/leader_elect/shard1/election
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:43.894; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0006 type:delete cxid:0x4b zxid:0x5a txntype:-1 reqpath:n/a Error Path:/solr/collections/collection1/leaders Error:KeeperErrorCode = NoNode for /solr/collections/collection1/leaders
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:43.895; org.apache.solr.cloud.ShardLeaderElectionContext; Running the leader process.
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:43.896; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0006 type:create cxid:0x4c zxid:0x5b txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:43.900; org.apache.solr.cloud.ShardLeaderElectionContext; Enough replicas found to continue.
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:43.900; org.apache.solr.cloud.ShardLeaderElectionContext; I may be the new leader - try and sync
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:43.901; org.apache.solr.cloud.SyncStrategy; Sync replicas to http://127.0.0.1:54571/collection1/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:43.901; org.apache.solr.cloud.SyncStrategy; Sync Success - now sync replicas to me
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:43.901; org.apache.solr.cloud.SyncStrategy; http://127.0.0.1:54571/collection1/ has no replicas
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:43.901; org.apache.solr.cloud.ShardLeaderElectionContext; I am the new leader: http://127.0.0.1:54571/collection1/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:43.902; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1/leaders/shard1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:43.905; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0006 type:create cxid:0x57 zxid:0x5f txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.420; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.435; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.436; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.436; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.460; org.apache.solr.cloud.ZkController; We are http://127.0.0.1:54571/collection1/ and leader is http://127.0.0.1:54571/collection1/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.460; org.apache.solr.cloud.ZkController; No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54571
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.460; org.apache.solr.cloud.ZkController; I am the leader, no recovery necessary
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.461; org.apache.solr.cloud.ZkController; publishing core=collection1 state=active
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.461; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.463; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.464; org.apache.solr.servlet.SolrDispatchFilter; user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.465; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init() done
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.465; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.570; org.eclipse.jetty.server.Server; jetty-8.1.8.v20121106
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.573; org.eclipse.jetty.server.AbstractConnector; Started SelectChannelConnector@127.0.0.1:58204
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.574; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init()
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.575; org.apache.solr.core.SolrResourceLoader; JNDI not configured for solr (NoInitialContextEx)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.575; org.apache.solr.core.SolrResourceLoader; using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.576; org.apache.solr.core.CoreContainer$Initializer; looking for solr config file: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/solr.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.576; org.apache.solr.core.CoreContainer; New CoreContainer 22798243
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.577; org.apache.solr.core.CoreContainer; Loading CoreContainer using Solr Home: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.577; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.619; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting socketTimeout to: 120000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.620; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting urlScheme to: http://
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.621; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting connTimeout to: 15000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.622; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxConnectionsPerHost to: 20
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.622; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting corePoolSize to: 0
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.623; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maximumPoolSize to: 2147483647
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.623; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxThreadIdleTime to: 5
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.624; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting sizeOfQueue to: -1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.624; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting fairnessPolicy to: false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.625; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.634; org.apache.solr.core.CoreContainer; Registering Log Listener
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.660; org.apache.solr.core.CoreContainer; Zookeeper client=127.0.0.1:58945/solr
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.660; org.apache.solr.cloud.ZkController; zkHost includes chroot
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.661; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945 sessionTimeout=60000 watcher=org.apache.solr.common.cloud.ConnectionManager@11da345
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.662; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.663; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.662; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.663; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49326
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.665; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49326
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.666; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0007 with negotiated timeout 20000 for client /127.0.0.1:49326
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.666; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0007, negotiated timeout = 20000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.666; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@11da345 name:ZooKeeperConnection Watcher:127.0.0.1:58945 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.667; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.668; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd0007
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.668; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd0007 closed
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.669; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.669; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49326 which had sessionid 0x13dc77a08cd0007
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.669; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.673; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=30000 watcher=org.apache.solr.common.cloud.ConnectionManager@9ce19f
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.679; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.680; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49327
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.680; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.681; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.682; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49327
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.684; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0008 with negotiated timeout 20000 for client /127.0.0.1:49327
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.684; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0008, negotiated timeout = 20000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.684; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@9ce19f name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.685; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.686; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0008 type:create cxid:0x1 zxid:0x6c txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.688; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0008 type:create cxid:0x3 zxid:0x6d txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:44.691; org.apache.solr.common.cloud.ZkStateReader; Updating cluster state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.695; org.apache.solr.cloud.ZkController; Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58204_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.696; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0008 type:delete cxid:0xd zxid:0x6e txntype:-1 reqpath:n/a Error Path:/solr/live_nodes/127.0.0.1:58204_ Error:KeeperErrorCode = NoNode for /solr/live_nodes/127.0.0.1:58204_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.697; org.apache.solr.common.cloud.SolrZkClient; makePath: /live_nodes/127.0.0.1:58204_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.699; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.699; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.701; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.701; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.701; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.703; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.703; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.707; org.apache.solr.core.CoreContainer; Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.707; org.apache.solr.cloud.ZkController; Check for collection zkNode:collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.717; org.apache.solr.cloud.ZkController; Collection zkNode exists
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.718; org.apache.solr.cloud.ZkController; Load collection config from:/collections/collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.720; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/collection1/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.720; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/collection1/lib/README' to classloader
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.720; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/collection1/lib/classes/' to classloader
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.757; org.apache.solr.core.SolrConfig; Using Lucene MatchVersion: LUCENE_50
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.796; org.apache.solr.core.SolrConfig; Loaded SolrConfig: solrconfig.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.797; org.apache.solr.schema.IndexSchema; Reading Solr Schema from schema.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.803; org.apache.solr.schema.IndexSchema; Schema name=test
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.942; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.943; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
> [junit4:junit4] 1> "operation":"state",
> [junit4:junit4] 1> "core_node_name":"2",
> [junit4:junit4] 1> "numShards":"3",
> [junit4:junit4] 1> "shard":"shard1",
> [junit4:junit4] 1> "roles":null,
> [junit4:junit4] 1> "state":"active",
> [junit4:junit4] 1> "core":"collection1",
> [junit4:junit4] 1> "collection":"collection1",
> [junit4:junit4] 1> "node_name":"127.0.0.1:54571_",
> [junit4:junit4] 1> "base_url":"http://127.0.0.1:54571"}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.963; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.963; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.964; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:45.963; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.361; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.368; org.apache.solr.schema.IndexSchema; default search field in schema is text
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.371; org.apache.solr.schema.IndexSchema; unique key field: id
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.390; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.393; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.397; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:46.398; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:46.399; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.400; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:46.401; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:46.401; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.402; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.402; org.apache.solr.core.SolrCore; [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1364851304470/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty2/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.402; org.apache.solr.core.JmxMonitoredMap; JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f58c60
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.403; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.403; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty2
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.403; org.apache.solr.core.SolrCore; New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty2/index/
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:46.404; org.apache.solr.core.SolrCore; [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty2/index' doesn't exist. Creating new index...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.404; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty2/index
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.407; org.apache.solr.core.SolrDeletionPolicy; SolrDeletionPolicy.onCommit: commits:num=1
> [junit4:junit4] 1> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty2/index,segFN=segments_1,generation=1,filenames=[segments_1]
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.408; org.apache.solr.core.SolrDeletionPolicy; newest commit = 1[segments_1]
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.410; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.410; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.411; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.411; org.apache.solr.core.RequestHandlers; created standard: solr.StandardRequestHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.412; org.apache.solr.core.RequestHandlers; created /get: solr.RealTimeGetHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.412; org.apache.solr.core.RequestHandlers; adding lazy requestHandler: solr.ReplicationHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.412; org.apache.solr.core.RequestHandlers; created /replication: solr.ReplicationHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.412; org.apache.solr.core.RequestHandlers; created /update: solr.UpdateRequestHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.413; org.apache.solr.core.RequestHandlers; created /admin/: org.apache.solr.handler.admin.AdminHandlers
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.416; org.apache.solr.handler.loader.XMLLoader; xsltCacheLifetimeSeconds=60
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.419; org.apache.solr.search.SolrIndexSearcher; Opening Searcher@104284e main
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.420; org.apache.solr.update.CommitTracker; Hard AutoCommit: disabled
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.420; org.apache.solr.update.CommitTracker; Soft AutoCommit: disabled
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.425; org.apache.solr.core.SolrCore; [collection1] Registered new searcher Searcher@104284e main{StandardDirectoryReader(segments_1:1)}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.427; org.apache.solr.cloud.ZkController; publishing core=collection1 state=down
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:46.427; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:47.468; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:47.469; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
> [junit4:junit4] 1> "operation":"state",
> [junit4:junit4] 1> "core_node_name":"3",
> [junit4:junit4] 1> "numShards":"3",
> [junit4:junit4] 1> "shard":null,
> [junit4:junit4] 1> "roles":null,
> [junit4:junit4] 1> "state":"down",
> [junit4:junit4] 1> "core":"collection1",
> [junit4:junit4] 1> "collection":"collection1",
> [junit4:junit4] 1> "node_name":"127.0.0.1:58204_",
> [junit4:junit4] 1> "base_url":"http://127.0.0.1:58204"}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:47.470; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Collection already exists with numShards=3
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:47.470; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Assigning new node to shard shard=shard2
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:47.475; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:47.475; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:47.475; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:47.475; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.430; org.apache.solr.core.CoreContainer; registering core: collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.431; org.apache.solr.cloud.ZkController; Register replica - core:collection1 address:http://127.0.0.1:58204 collection:collection1 shard:shard2
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.432; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1/leader_elect/shard2/election
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.441; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0008 type:delete cxid:0x4a zxid:0x7d txntype:-1 reqpath:n/a Error Path:/solr/collections/collection1/leaders/shard2 Error:KeeperErrorCode = NoNode for /solr/collections/collection1/leaders/shard2
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.442; org.apache.solr.cloud.ShardLeaderElectionContext; Running the leader process.
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.445; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0008 type:create cxid:0x4b zxid:0x7e txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.447; org.apache.solr.cloud.ShardLeaderElectionContext; Enough replicas found to continue.
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.448; org.apache.solr.cloud.ShardLeaderElectionContext; I may be the new leader - try and sync
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.448; org.apache.solr.cloud.SyncStrategy; Sync replicas to http://127.0.0.1:58204/collection1/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.448; org.apache.solr.cloud.SyncStrategy; Sync Success - now sync replicas to me
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.448; org.apache.solr.cloud.SyncStrategy; http://127.0.0.1:58204/collection1/ has no replicas
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.449; org.apache.solr.cloud.ShardLeaderElectionContext; I am the new leader: http://127.0.0.1:58204/collection1/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.449; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1/leaders/shard2
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.453; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd0008 type:create cxid:0x55 zxid:0x81 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.980; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.988; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.989; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.989; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:48.989; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.006; org.apache.solr.cloud.ZkController; We are http://127.0.0.1:58204/collection1/ and leader is http://127.0.0.1:58204/collection1/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.007; org.apache.solr.cloud.ZkController; No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:58204
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.007; org.apache.solr.cloud.ZkController; I am the leader, no recovery necessary
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.008; org.apache.solr.cloud.ZkController; publishing core=collection1 state=active
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.008; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.010; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.011; org.apache.solr.servlet.SolrDispatchFilter; user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.011; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init() done
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.012; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.109; org.eclipse.jetty.server.Server; jetty-8.1.8.v20121106
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.113; org.eclipse.jetty.server.AbstractConnector; Started SelectChannelConnector@127.0.0.1:46444
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.114; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init()
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.114; org.apache.solr.core.SolrResourceLoader; JNDI not configured for solr (NoInitialContextEx)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.114; org.apache.solr.core.SolrResourceLoader; using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.115; org.apache.solr.core.CoreContainer$Initializer; looking for solr config file: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/solr.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.115; org.apache.solr.core.CoreContainer; New CoreContainer 22206042
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.116; org.apache.solr.core.CoreContainer; Loading CoreContainer using Solr Home: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.116; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.137; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting socketTimeout to: 120000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.139; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting urlScheme to: http://
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.139; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting connTimeout to: 15000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.140; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxConnectionsPerHost to: 20
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.140; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting corePoolSize to: 0
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.141; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maximumPoolSize to: 2147483647
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.141; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxThreadIdleTime to: 5
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.142; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting sizeOfQueue to: -1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.142; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting fairnessPolicy to: false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.143; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.151; org.apache.solr.core.CoreContainer; Registering Log Listener
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.165; org.apache.solr.core.CoreContainer; Zookeeper client=127.0.0.1:58945/solr
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.165; org.apache.solr.cloud.ZkController; zkHost includes chroot
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.166; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945 sessionTimeout=60000 watcher=org.apache.solr.common.cloud.ConnectionManager@17042f6
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.167; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.177; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.178; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.177; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49328
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.179; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49328
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.188; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd0009 with negotiated timeout 20000 for client /127.0.0.1:49328
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.188; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd0009, negotiated timeout = 20000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.190; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@17042f6 name:ZooKeeperConnection Watcher:127.0.0.1:58945 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.190; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.191; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd0009
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.192; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd0009 closed
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.192; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.192; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49328 which had sessionid 0x13dc77a08cd0009
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.193; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.199; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=30000 watcher=org.apache.solr.common.cloud.ConnectionManager@549c55
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.200; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.200; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49329
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.200; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.201; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49329
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.202; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd000a with negotiated timeout 20000 for client /127.0.0.1:49329
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.202; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd000a, negotiated timeout = 20000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.204; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.204; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@549c55 name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.204; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.205; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000a type:create cxid:0x1 zxid:0x8e txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.206; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000a type:create cxid:0x3 zxid:0x8f txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:49.208; org.apache.solr.common.cloud.ZkStateReader; Updating cluster state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.211; org.apache.solr.cloud.ZkController; Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46444_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.213; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000a type:delete cxid:0xd zxid:0x90 txntype:-1 reqpath:n/a Error Path:/solr/live_nodes/127.0.0.1:46444_ Error:KeeperErrorCode = NoNode for /solr/live_nodes/127.0.0.1:46444_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.214; org.apache.solr.common.cloud.SolrZkClient; makePath: /live_nodes/127.0.0.1:46444_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.216; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.217; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.217; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.218; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.218; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.219; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.220; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.220; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.227; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.231; org.apache.solr.core.CoreContainer; Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.232; org.apache.solr.cloud.ZkController; Check for collection zkNode:collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.232; org.apache.solr.cloud.ZkController; Collection zkNode exists
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.232; org.apache.solr.cloud.ZkController; Load collection config from:/collections/collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.233; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/collection1/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.234; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/collection1/lib/README' to classloader
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.234; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/collection1/lib/classes/' to classloader
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.261; org.apache.solr.core.SolrConfig; Using Lucene MatchVersion: LUCENE_50
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.301; org.apache.solr.core.SolrConfig; Loaded SolrConfig: solrconfig.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.303; org.apache.solr.schema.IndexSchema; Reading Solr Schema from schema.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.308; org.apache.solr.schema.IndexSchema; Schema name=test
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.497; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.498; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
> [junit4:junit4] 1> "operation":"state",
> [junit4:junit4] 1> "core_node_name":"3",
> [junit4:junit4] 1> "numShards":"3",
> [junit4:junit4] 1> "shard":"shard2",
> [junit4:junit4] 1> "roles":null,
> [junit4:junit4] 1> "state":"active",
> [junit4:junit4] 1> "core":"collection1",
> [junit4:junit4] 1> "collection":"collection1",
> [junit4:junit4] 1> "node_name":"127.0.0.1:58204_",
> [junit4:junit4] 1> "base_url":"http://127.0.0.1:58204"}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.501; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.502; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.502; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.502; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.503; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.829; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.835; org.apache.solr.schema.IndexSchema; default search field in schema is text
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.837; org.apache.solr.schema.IndexSchema; unique key field: id
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.850; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.857; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.861; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:50.863; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:50.864; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.864; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:50.865; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:50.866; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.866; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.866; org.apache.solr.core.SolrCore; [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1364851309015/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty3/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.867; org.apache.solr.core.JmxMonitoredMap; JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f58c60
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.867; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.868; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty3
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.868; org.apache.solr.core.SolrCore; New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty3/index/
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:50.869; org.apache.solr.core.SolrCore; [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty3/index' doesn't exist. Creating new index...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.869; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty3/index
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.873; org.apache.solr.core.SolrDeletionPolicy; SolrDeletionPolicy.onCommit: commits:num=1
> [junit4:junit4] 1> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty3/index,segFN=segments_1,generation=1,filenames=[segments_1]
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.873; org.apache.solr.core.SolrDeletionPolicy; newest commit = 1[segments_1]
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.875; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.875; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.876; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.877; org.apache.solr.core.RequestHandlers; created standard: solr.StandardRequestHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.877; org.apache.solr.core.RequestHandlers; created /get: solr.RealTimeGetHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.878; org.apache.solr.core.RequestHandlers; adding lazy requestHandler: solr.ReplicationHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.878; org.apache.solr.core.RequestHandlers; created /replication: solr.ReplicationHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.879; org.apache.solr.core.RequestHandlers; created /update: solr.UpdateRequestHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.879; org.apache.solr.core.RequestHandlers; created /admin/: org.apache.solr.handler.admin.AdminHandlers
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.883; org.apache.solr.handler.loader.XMLLoader; xsltCacheLifetimeSeconds=60
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.892; org.apache.solr.search.SolrIndexSearcher; Opening Searcher@ad0057 main
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.893; org.apache.solr.update.CommitTracker; Hard AutoCommit: disabled
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.893; org.apache.solr.update.CommitTracker; Soft AutoCommit: disabled
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.899; org.apache.solr.cloud.ZkController; publishing core=collection1 state=down
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.899; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:50.902; org.apache.solr.core.SolrCore; [collection1] Registered new searcher Searcher@ad0057 main{StandardDirectoryReader(segments_1:1)}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.009; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.010; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
> [junit4:junit4] 1> "operation":"state",
> [junit4:junit4] 1> "core_node_name":"4",
> [junit4:junit4] 1> "numShards":"3",
> [junit4:junit4] 1> "shard":null,
> [junit4:junit4] 1> "roles":null,
> [junit4:junit4] 1> "state":"down",
> [junit4:junit4] 1> "core":"collection1",
> [junit4:junit4] 1> "collection":"collection1",
> [junit4:junit4] 1> "node_name":"127.0.0.1:46444_",
> [junit4:junit4] 1> "base_url":"http://127.0.0.1:46444"}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.010; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Collection already exists with numShards=3
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.011; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Assigning new node to shard shard=shard3
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.021; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.021; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.021; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.021; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.021; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.902; org.apache.solr.core.CoreContainer; registering core: collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.903; org.apache.solr.cloud.ZkController; Register replica - core:collection1 address:http://127.0.0.1:46444 collection:collection1 shard:shard3
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.906; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1/leader_elect/shard3/election
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.915; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000a type:delete cxid:0x4a zxid:0x9f txntype:-1 reqpath:n/a Error Path:/solr/collections/collection1/leaders/shard3 Error:KeeperErrorCode = NoNode for /solr/collections/collection1/leaders/shard3
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.916; org.apache.solr.cloud.ShardLeaderElectionContext; Running the leader process.
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.917; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000a type:create cxid:0x4b zxid:0xa0 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.920; org.apache.solr.cloud.ShardLeaderElectionContext; Enough replicas found to continue.
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.920; org.apache.solr.cloud.ShardLeaderElectionContext; I may be the new leader - try and sync
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.920; org.apache.solr.cloud.SyncStrategy; Sync replicas to http://127.0.0.1:46444/collection1/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.920; org.apache.solr.cloud.SyncStrategy; Sync Success - now sync replicas to me
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.921; org.apache.solr.cloud.SyncStrategy; http://127.0.0.1:46444/collection1/ has no replicas
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.921; org.apache.solr.cloud.ShardLeaderElectionContext; I am the new leader: http://127.0.0.1:46444/collection1/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.921; org.apache.solr.common.cloud.SolrZkClient; makePath: /collections/collection1/leaders/shard3
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:52.934; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000a type:create cxid:0x55 zxid:0xa3 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.527; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.540; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.540; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.540; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.540; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.545; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.547; org.apache.solr.cloud.ZkController; We are http://127.0.0.1:46444/collection1/ and leader is http://127.0.0.1:46444/collection1/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.547; org.apache.solr.cloud.ZkController; No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:46444
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.548; org.apache.solr.cloud.ZkController; I am the leader, no recovery necessary
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.548; org.apache.solr.cloud.ZkController; publishing core=collection1 state=active
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.548; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.549; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.551; org.apache.solr.servlet.SolrDispatchFilter; user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.552; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init() done
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.552; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.649; org.eclipse.jetty.server.Server; jetty-8.1.8.v20121106
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.651; org.eclipse.jetty.server.AbstractConnector; Started SelectChannelConnector@127.0.0.1:44384
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.652; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init()
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.652; org.apache.solr.core.SolrResourceLoader; JNDI not configured for solr (NoInitialContextEx)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.653; org.apache.solr.core.SolrResourceLoader; using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.653; org.apache.solr.core.CoreContainer$Initializer; looking for solr config file: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/solr.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.654; org.apache.solr.core.CoreContainer; New CoreContainer 29587082
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.655; org.apache.solr.core.CoreContainer; Loading CoreContainer using Solr Home: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.655; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.687; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting socketTimeout to: 120000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.688; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting urlScheme to: http://
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.688; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting connTimeout to: 15000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.689; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxConnectionsPerHost to: 20
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.689; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting corePoolSize to: 0
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.690; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maximumPoolSize to: 2147483647
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.690; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxThreadIdleTime to: 5
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.691; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting sizeOfQueue to: -1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.692; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting fairnessPolicy to: false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.692; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.702; org.apache.solr.core.CoreContainer; Registering Log Listener
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.712; org.apache.solr.core.CoreContainer; Zookeeper client=127.0.0.1:58945/solr
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.712; org.apache.solr.cloud.ZkController; zkHost includes chroot
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.713; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945 sessionTimeout=60000 watcher=org.apache.solr.common.cloud.ConnectionManager@1fe7650
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.715; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.717; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.719; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49330
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.720; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.720; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49330
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.721; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd000b with negotiated timeout 20000 for client /127.0.0.1:49330
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.722; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd000b, negotiated timeout = 20000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.722; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@1fe7650 name:ZooKeeperConnection Watcher:127.0.0.1:58945 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.722; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.732; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd000b
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.732; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49330 which had sessionid 0x13dc77a08cd000b
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.736; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd000b closed
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.737; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.738; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.742; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=30000 watcher=org.apache.solr.common.cloud.ConnectionManager@e6a671
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.744; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.744; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.744; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49331
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.745; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.745; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49331
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.746; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd000c with negotiated timeout 20000 for client /127.0.0.1:49331
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.747; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd000c, negotiated timeout = 20000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.748; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@e6a671 name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.749; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.750; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000c type:create cxid:0x1 zxid:0xb0 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.752; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000c type:create cxid:0x3 zxid:0xb1 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:53.754; org.apache.solr.common.cloud.ZkStateReader; Updating cluster state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.758; org.apache.solr.cloud.ZkController; Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44384_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.760; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000c type:delete cxid:0xd zxid:0xb2 txntype:-1 reqpath:n/a Error Path:/solr/live_nodes/127.0.0.1:44384_ Error:KeeperErrorCode = NoNode for /solr/live_nodes/127.0.0.1:44384_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.761; org.apache.solr.common.cloud.SolrZkClient; makePath: /live_nodes/127.0.0.1:44384_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.768; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.768; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.771; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.772; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.772; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.768; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.769; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.773; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.773; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.775; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.776; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.790; org.apache.solr.core.CoreContainer; Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.791; org.apache.solr.cloud.ZkController; Check for collection zkNode:collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.792; org.apache.solr.cloud.ZkController; Collection zkNode exists
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.792; org.apache.solr.cloud.ZkController; Load collection config from:/collections/collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.794; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/collection1/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.794; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/collection1/lib/README' to classloader
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.794; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/collection1/lib/classes/' to classloader
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.834; org.apache.solr.core.SolrConfig; Using Lucene MatchVersion: LUCENE_50
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.880; org.apache.solr.core.SolrConfig; Loaded SolrConfig: solrconfig.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.882; org.apache.solr.schema.IndexSchema; Reading Solr Schema from schema.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:54.888; org.apache.solr.schema.IndexSchema; Schema name=test
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.049; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.050; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
> [junit4:junit4] 1> "operation":"state",
> [junit4:junit4] 1> "core_node_name":"4",
> [junit4:junit4] 1> "numShards":"3",
> [junit4:junit4] 1> "shard":"shard3",
> [junit4:junit4] 1> "roles":null,
> [junit4:junit4] 1> "state":"active",
> [junit4:junit4] 1> "core":"collection1",
> [junit4:junit4] 1> "collection":"collection1",
> [junit4:junit4] 1> "node_name":"127.0.0.1:46444_",
> [junit4:junit4] 1> "base_url":"http://127.0.0.1:46444"}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.055; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.056; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.055; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.055; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.056; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.055; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.460; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.467; org.apache.solr.schema.IndexSchema; default search field in schema is text
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.470; org.apache.solr.schema.IndexSchema; unique key field: id
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.487; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.491; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.495; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:55.498; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:55.499; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.499; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:55.500; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:55.500; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.501; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.501; org.apache.solr.core.SolrCore; [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty4-1364851313556/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty4/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.501; org.apache.solr.core.JmxMonitoredMap; JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f58c60
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.502; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.502; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty4
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.502; org.apache.solr.core.SolrCore; New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty4/index/
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:55.503; org.apache.solr.core.SolrCore; [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty4/index' doesn't exist. Creating new index...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.503; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty4/index
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.506; org.apache.solr.core.SolrDeletionPolicy; SolrDeletionPolicy.onCommit: commits:num=1
> [junit4:junit4] 1> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty4/index,segFN=segments_1,generation=1,filenames=[segments_1]
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.506; org.apache.solr.core.SolrDeletionPolicy; newest commit = 1[segments_1]
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.509; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.509; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.510; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.510; org.apache.solr.core.RequestHandlers; created standard: solr.StandardRequestHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.511; org.apache.solr.core.RequestHandlers; created /get: solr.RealTimeGetHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.511; org.apache.solr.core.RequestHandlers; adding lazy requestHandler: solr.ReplicationHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.512; org.apache.solr.core.RequestHandlers; created /replication: solr.ReplicationHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.512; org.apache.solr.core.RequestHandlers; created /update: solr.UpdateRequestHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.513; org.apache.solr.core.RequestHandlers; created /admin/: org.apache.solr.handler.admin.AdminHandlers
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.516; org.apache.solr.handler.loader.XMLLoader; xsltCacheLifetimeSeconds=60
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.519; org.apache.solr.search.SolrIndexSearcher; Opening Searcher@f80443 main
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.520; org.apache.solr.update.CommitTracker; Hard AutoCommit: disabled
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.520; org.apache.solr.update.CommitTracker; Soft AutoCommit: disabled
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.526; org.apache.solr.core.SolrCore; [collection1] Registered new searcher Searcher@f80443 main{StandardDirectoryReader(segments_1:1)}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.528; org.apache.solr.cloud.ZkController; publishing core=collection1 state=down
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:55.528; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:56.561; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:56.562; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
> [junit4:junit4] 1> "operation":"state",
> [junit4:junit4] 1> "core_node_name":"5",
> [junit4:junit4] 1> "numShards":"3",
> [junit4:junit4] 1> "shard":null,
> [junit4:junit4] 1> "roles":null,
> [junit4:junit4] 1> "state":"down",
> [junit4:junit4] 1> "core":"collection1",
> [junit4:junit4] 1> "collection":"collection1",
> [junit4:junit4] 1> "node_name":"127.0.0.1:44384_",
> [junit4:junit4] 1> "base_url":"http://127.0.0.1:44384"}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:56.562; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Collection already exists with numShards=3
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:56.562; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Assigning new node to shard shard=shard1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:56.566; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:56.567; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:56.567; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:56.567; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:56.567; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:56.567; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.529; org.apache.solr.core.CoreContainer; registering core: collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.530; org.apache.solr.cloud.ZkController; Register replica - core:collection1 address:http://127.0.0.1:44384 collection:collection1 shard:shard1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.533; org.apache.solr.cloud.ZkController; We are http://127.0.0.1:44384/collection1/ and leader is http://127.0.0.1:54571/collection1/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.533; org.apache.solr.cloud.ZkController; No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:44384
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.534; org.apache.solr.cloud.ZkController; Core needs to recover:collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.534; org.apache.solr.update.DefaultSolrCoreState; Running recovery - first canceling any ongoing recovery
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.534; org.apache.solr.cloud.RecoveryStrategy; Starting recovery process. core=collection1 recoveringAfterStartup=true
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.535; org.apache.solr.cloud.RecoveryStrategy; ###### startupVersions=[]
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.535; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.535; org.apache.solr.cloud.ZkController; publishing core=collection1 state=recovering
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.535; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.536; org.apache.solr.servlet.SolrDispatchFilter; user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.536; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init() done
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.537; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.584; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.597; org.eclipse.jetty.server.Server; jetty-8.1.8.v20121106
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.599; org.eclipse.jetty.server.AbstractConnector; Started SelectChannelConnector@127.0.0.1:43323
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.600; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init()
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.600; org.apache.solr.core.SolrResourceLoader; JNDI not configured for solr (NoInitialContextEx)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.600; org.apache.solr.core.SolrResourceLoader; using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.601; org.apache.solr.core.CoreContainer$Initializer; looking for solr config file: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/solr.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.601; org.apache.solr.core.CoreContainer; New CoreContainer 13718597
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.601; org.apache.solr.core.CoreContainer; Loading CoreContainer using Solr Home: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.602; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.621; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting socketTimeout to: 120000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.622; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting urlScheme to: http://
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.622; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting connTimeout to: 15000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.623; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxConnectionsPerHost to: 20
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.623; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting corePoolSize to: 0
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.623; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maximumPoolSize to: 2147483647
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.624; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting maxThreadIdleTime to: 5
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.624; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting sizeOfQueue to: -1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.624; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting fairnessPolicy to: false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.625; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.630; org.apache.solr.core.CoreContainer; Registering Log Listener
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.640; org.apache.solr.core.CoreContainer; Zookeeper client=127.0.0.1:58945/solr
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.641; org.apache.solr.cloud.ZkController; zkHost includes chroot
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.641; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945 sessionTimeout=60000 watcher=org.apache.solr.common.cloud.ConnectionManager@9f2657
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.642; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.642; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.643; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.643; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49334
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.644; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49334
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.645; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd000d with negotiated timeout 20000 for client /127.0.0.1:49334
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.645; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd000d, negotiated timeout = 20000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.645; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@9f2657 name:ZooKeeperConnection Watcher:127.0.0.1:58945 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.646; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.648; org.apache.zookeeper.server.PrepRequestProcessor; Processed session termination for sessionid: 0x13dc77a08cd000d
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.648; org.apache.zookeeper.ZooKeeper; Session: 0x13dc77a08cd000d closed
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.648; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.648; org.apache.zookeeper.server.NIOServerCnxn; Closed socket connection for client /127.0.0.1:49334 which had sessionid 0x13dc77a08cd000d
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.649; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.652; org.apache.zookeeper.ZooKeeper; Initiating client connection, connectString=127.0.0.1:58945/solr sessionTimeout=30000 watcher=org.apache.solr.common.cloud.ConnectionManager@ea502d
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.653; org.apache.solr.common.cloud.ConnectionManager; Waiting for client to connect to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.654; org.apache.zookeeper.ClientCnxn$SendThread; Opening socket connection to server localhost.localdomain/127.0.0.1:58945. Will not attempt to authenticate using SASL (access denied ("javax.security.auth.AuthPermission" "getLoginConfiguration"))
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.654; org.apache.zookeeper.ClientCnxn$SendThread; Socket connection established to localhost.localdomain/127.0.0.1:58945, initiating session
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.654; org.apache.zookeeper.server.NIOServerCnxnFactory; Accepted socket connection from /127.0.0.1:49335
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.655; org.apache.zookeeper.server.ZooKeeperServer; Client attempting to establish new session at /127.0.0.1:49335
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.656; org.apache.zookeeper.server.ZooKeeperServer; Established session 0x13dc77a08cd000e with negotiated timeout 20000 for client /127.0.0.1:49335
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.656; org.apache.zookeeper.ClientCnxn$SendThread; Session establishment complete on server localhost.localdomain/127.0.0.1:58945, sessionid = 0x13dc77a08cd000e, negotiated timeout = 20000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.656; org.apache.solr.common.cloud.ConnectionManager; Watcher org.apache.solr.common.cloud.ConnectionManager@ea502d name:ZooKeeperConnection Watcher:127.0.0.1:58945/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.656; org.apache.solr.common.cloud.ConnectionManager; Client is connected to ZooKeeper
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.657; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000e type:create cxid:0x1 zxid:0xc3 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.659; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000e type:create cxid:0x3 zxid:0xc4 txntype:-1 reqpath:n/a Error Path:/solr/overseer Error:KeeperErrorCode = NodeExists for /solr/overseer
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:57.660; org.apache.solr.common.cloud.ZkStateReader; Updating cluster state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.073; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.074; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
> [junit4:junit4] 1> "operation":"state",
> [junit4:junit4] 1> "core_node_name":"5",
> [junit4:junit4] 1> "numShards":"3",
> [junit4:junit4] 1> "shard":"shard1",
> [junit4:junit4] 1> "roles":null,
> [junit4:junit4] 1> "state":"recovering",
> [junit4:junit4] 1> "core":"collection1",
> [junit4:junit4] 1> "collection":"collection1",
> [junit4:junit4] 1> "node_name":"127.0.0.1:44384_",
> [junit4:junit4] 1> "base_url":"http://127.0.0.1:44384"}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.077; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.077; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.078; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.077; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.077; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.077; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.077; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.589; org.apache.solr.servlet.SolrDispatchFilter; [admin] webapp=null path=/admin/cores params={coreNodeName=5&state=recovering&nodeName=127.0.0.1:44384_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1000
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.663; org.apache.solr.cloud.ZkController; Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43323_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.664; org.apache.zookeeper.server.PrepRequestProcessor; Got user-level KeeperException when processing sessionid:0x13dc77a08cd000e type:delete cxid:0xf zxid:0xc9 txntype:-1 reqpath:n/a Error Path:/solr/live_nodes/127.0.0.1:43323_ Error:KeeperErrorCode = NoNode for /solr/live_nodes/127.0.0.1:43323_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.665; org.apache.solr.common.cloud.SolrZkClient; makePath: /live_nodes/127.0.0.1:43323_
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.667; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.667; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.668; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.667; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.668; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.668; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.668; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.669; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.669; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.669; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.670; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.670; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.670; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.671; org.apache.solr.common.cloud.ZkStateReader$3; Updating live nodes... (6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.676; org.apache.solr.core.CoreContainer; Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.676; org.apache.solr.cloud.ZkController; Check for collection zkNode:collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.677; org.apache.solr.cloud.ZkController; Collection zkNode exists
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.677; org.apache.solr.cloud.ZkController; Load collection config from:/collections/collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.678; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/collection1/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.678; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/collection1/lib/README' to classloader
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.679; org.apache.solr.core.SolrResourceLoader; Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/collection1/lib/classes/' to classloader
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.702; org.apache.solr.core.SolrConfig; Using Lucene MatchVersion: LUCENE_50
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.731; org.apache.solr.core.SolrConfig; Loaded SolrConfig: solrconfig.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.732; org.apache.solr.schema.IndexSchema; Reading Solr Schema from schema.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:58.736; org.apache.solr.schema.IndexSchema; Schema name=test
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.092; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.097; org.apache.solr.schema.IndexSchema; default search field in schema is text
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.100; org.apache.solr.schema.IndexSchema; unique key field: id
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.112; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.115; org.apache.solr.schema.FileExchangeRateProvider; Reloading exchange rates from file currency.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.118; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:59.119; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:59.120; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.120; org.apache.solr.schema.OpenExchangeRatesOrgProvider; Reloading exchange rates from open-exchange-rates.json
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:59.121; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Unknown key IMPORTANT NOTE
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:59.121; org.apache.solr.schema.OpenExchangeRatesOrgProvider$OpenExchangeRates; Expected key, got STRING
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.121; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.122; org.apache.solr.core.SolrCore; [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty5-1364851317540/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty5/
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.122; org.apache.solr.core.JmxMonitoredMap; JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f58c60
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.122; org.apache.solr.core.SolrCore; solr.StandardDirectoryFactory
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.123; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty5
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.123; org.apache.solr.core.SolrCore; New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty5/index/
> [junit4:junit4] 1> WARN - 2013-04-01 21:21:59.123; org.apache.solr.core.SolrCore; [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty5/index' doesn't exist. Creating new index...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.123; org.apache.solr.core.CachingDirectoryFactory; return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty5/index
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.225; org.apache.solr.core.SolrDeletionPolicy; SolrDeletionPolicy.onCommit: commits:num=1
> [junit4:junit4] 1> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1364851296457/jetty5/index,segFN=segments_1,generation=1,filenames=[segments_1]
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.226; org.apache.solr.core.SolrDeletionPolicy; newest commit = 1[segments_1]
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.227; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.227; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.228; org.apache.solr.update.processor.UpdateRequestProcessorChain; inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.228; org.apache.solr.core.RequestHandlers; created standard: solr.StandardRequestHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.229; org.apache.solr.core.RequestHandlers; created /get: solr.RealTimeGetHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.229; org.apache.solr.core.RequestHandlers; adding lazy requestHandler: solr.ReplicationHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.229; org.apache.solr.core.RequestHandlers; created /replication: solr.ReplicationHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.230; org.apache.solr.core.RequestHandlers; created /update: solr.UpdateRequestHandler
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.230; org.apache.solr.core.RequestHandlers; created /admin/: org.apache.solr.handler.admin.AdminHandlers
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.233; org.apache.solr.handler.loader.XMLLoader; xsltCacheLifetimeSeconds=60
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.237; org.apache.solr.search.SolrIndexSearcher; Opening Searcher@186d3c7 main
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.238; org.apache.solr.update.CommitTracker; Hard AutoCommit: disabled
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.238; org.apache.solr.update.CommitTracker; Soft AutoCommit: disabled
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.241; org.apache.solr.core.SolrCore; [collection1] Registered new searcher Searcher@186d3c7 main{StandardDirectoryReader(segments_1:1)}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.243; org.apache.solr.cloud.ZkController; publishing core=collection1 state=down
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.243; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.583; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.584; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Update state numShards=3 message={
> [junit4:junit4] 1> "operation":"state",
> [junit4:junit4] 1> "core_node_name":"6",
> [junit4:junit4] 1> "numShards":"3",
> [junit4:junit4] 1> "shard":null,
> [junit4:junit4] 1> "roles":null,
> [junit4:junit4] 1> "state":"down",
> [junit4:junit4] 1> "core":"collection1",
> [junit4:junit4] 1> "collection":"collection1",
> [junit4:junit4] 1> "node_name":"127.0.0.1:43323_",
> [junit4:junit4] 1> "base_url":"http://127.0.0.1:43323"}
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.584; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Collection already exists with numShards=3
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.584; org.apache.solr.cloud.Overseer$ClusterStateUpdater; Assigning new node to shard shard=shard2
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.588; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.588; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.588; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.588; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.588; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.588; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:21:59.588; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.244; org.apache.solr.core.CoreContainer; registering core: collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.245; org.apache.solr.cloud.ZkController; Register replica - core:collection1 address:http://127.0.0.1:43323 collection:collection1 shard:shard2
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.247; org.apache.solr.cloud.ZkController; We are http://127.0.0.1:43323/collection1/ and leader is http://127.0.0.1:58204/collection1/
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.247; org.apache.solr.cloud.ZkController; No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:43323
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.247; org.apache.solr.cloud.ZkController; Core needs to recover:collection1
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.248; org.apache.solr.update.DefaultSolrCoreState; Running recovery - first canceling any ongoing recovery
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.248; org.apache.solr.cloud.RecoveryStrategy; Starting recovery process. core=collection1 recoveringAfterStartup=true
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.249; org.apache.solr.cloud.RecoveryStrategy; ###### startupVersions=[]
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.249; org.apache.solr.common.cloud.ZkStateReader; Updating cloud state from ZooKeeper...
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.249; org.apache.solr.cloud.ZkController; publishing core=collection1 state=recovering
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.249; org.apache.solr.cloud.ZkController; numShards not found on descriptor - reading it from system property
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.250; org.apache.solr.servlet.SolrDispatchFilter; user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.250; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.250; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init() done
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.251; org.apache.solr.client.solrj.impl.HttpClientUtil; Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.312; org.eclipse.jetty.server.Server; jetty-8.1.8.v20121106
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.314; org.eclipse.jetty.server.AbstractConnector; Started SelectChannelConnector@127.0.0.1:53921
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.314; org.apache.solr.servlet.SolrDispatchFilter; SolrDispatchFilter.init()
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.314; org.apache.solr.core.SolrResourceLoader; JNDI not configured for solr (NoInitialContextEx)
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.315; org.apache.solr.core.SolrResourceLoader; using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1364851320255
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.315; org.apache.solr.core.CoreContainer$Initializer; looking for solr config file: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1364851320255/solr.xml
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.316; org.apache.solr.core.CoreContainer; New CoreContainer 22838968
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.316; org.apache.solr.core.CoreContainer; Loading CoreContainer using Solr Home: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1364851320255/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.316; org.apache.solr.core.SolrResourceLoader; new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty6-1364851320255/'
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.335; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting socketTimeout to: 120000
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.335; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting urlScheme to: http://
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.335; org.apache.solr.handler.component.HttpShardHandlerFactory; Setting connTimeout to: 15000
> [junit4:junit4] 1> INFO - 2013-04-01 21:22:00.336; org.apache.solr.handler.componen
>
> [...truncated too long message...]
>
> jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./solrtest-ChaosMonkeySafeLeaderTest-1364851296457
> [junit4:junit4] 2> added docs:2216 with 0 fails deletes:1051
> [junit4:junit4] 2> added docs:2270 with 0 fails deletes:1079
> [junit4:junit4] 2> check const of shard1
> [junit4:junit4] 2> client0
> [junit4:junit4] 2> PROPS:2:{"shard":"shard1","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:54571_","base_url":"http://127.0.0.1:54571","leader":"true"}
> [junit4:junit4] 2> live:true
> [junit4:junit4] 2> num:775
> [junit4:junit4] 2>
> [junit4:junit4] 2> client1
> [junit4:junit4] 2> PROPS:5:{"shard":"shard1","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:44384_","base_url":"http://127.0.0.1:44384"}
> [junit4:junit4] 2> live:true
> [junit4:junit4] 2> num:775
> [junit4:junit4] 2>
> [junit4:junit4] 2> client2
> [junit4:junit4] 2> PROPS:8:{"shard":"shard1","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:42744_","base_url":"http://127.0.0.1:42744"}
> [junit4:junit4] 2> live:true
> [junit4:junit4] 2> num:775
> [junit4:junit4] 2>
> [junit4:junit4] 2> client3
> [junit4:junit4] 2> PROPS:11:{"shard":"shard1","state":"recovery_failed","core":"collection1","collection":"collection1","node_name":"127.0.0.1:47960_","base_url":"http://127.0.0.1:47960"}
> [junit4:junit4] 2> error contacting client: Server refused connection at: http://127.0.0.1:47960/collection1
> [junit4:junit4] 2>
> [junit4:junit4] 2> check const of shard2
> [junit4:junit4] 2> client0
> [junit4:junit4] 2> PROPS:3:{"shard":"shard2","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:58204_","base_url":"http://127.0.0.1:58204","leader":"true"}
> [junit4:junit4] 2> live:true
> [junit4:junit4] 2> num:785
> [junit4:junit4] 2>
> [junit4:junit4] 2> client1
> [junit4:junit4] 2> PROPS:6:{"shard":"shard2","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:43323_","base_url":"http://127.0.0.1:43323"}
> [junit4:junit4] 2> live:true
> [junit4:junit4] 2> num:785
> [junit4:junit4] 2>
> [junit4:junit4] 2> client2
> [junit4:junit4] 2> PROPS:9:{"shard":"shard2","state":"recovering","core":"collection1","collection":"collection1","node_name":"127.0.0.1:41132_","base_url":"http://127.0.0.1:41132"}
> [junit4:junit4] 2> live:false
> [junit4:junit4] 2> num:51
> [junit4:junit4] 2>
> [junit4:junit4] 2> client3
> [junit4:junit4] 2> PROPS:12:{"shard":"shard2","state":"down","core":"collection1","collection":"collection1","node_name":"127.0.0.1:42437_","base_url":"http://127.0.0.1:42437"}
> [junit4:junit4] 2> error contacting client: Server refused connection at: http://127.0.0.1:42437/collection1
> [junit4:junit4] 2>
> [junit4:junit4] 2> check const of shard3
> [junit4:junit4] 2> client0
> [junit4:junit4] 2> PROPS:4:{"shard":"shard3","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:46444_","base_url":"http://127.0.0.1:46444","leader":"true"}
> [junit4:junit4] 2> live:true
> [junit4:junit4] 2> num:797
> [junit4:junit4] 2>
> [junit4:junit4] 2> client1
> [junit4:junit4] 2> PROPS:7:{"shard":"shard3","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:53921_","base_url":"http://127.0.0.1:53921"}
> [junit4:junit4] 2> live:true
> [junit4:junit4] 2> num:797
> [junit4:junit4] 2>
> [junit4:junit4] 2> client2
> [junit4:junit4] 2> PROPS:10:{"shard":"shard3","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:51794_","base_url":"http://127.0.0.1:51794"}
> [junit4:junit4] 2> live:true
> [junit4:junit4] 2> num:797
> [junit4:junit4] 2>
> [junit4:junit4] 2> client3
> [junit4:junit4] 2> PROPS:13:{"shard":"shard3","state":"active","core":"collection1","collection":"collection1","node_name":"127.0.0.1:42485_","base_url":"http://127.0.0.1:42485"}
> [junit4:junit4] 2> live:true
> [junit4:junit4] 2> num:797
> [junit4:junit4] 2>
> [junit4:junit4] 2> http://127.0.0.1:54571/collection1/ : 775
> [junit4:junit4] 2> shard:shard1
> [junit4:junit4] 2> http://127.0.0.1:58204/collection1/ : 785
> [junit4:junit4] 2> shard:shard2
> [junit4:junit4] 2> http://127.0.0.1:46444/collection1/ : 797
> [junit4:junit4] 2> shard:shard3
> [junit4:junit4] 2> ######controlDocList: SolrDocumentList[sz=2356]=[SolrDocument{id=60002}, SolrDocument{id=10002}, SolrDocument{id=60003}, SolrDocument{id=10003}, SolrDocument{id=60007}] , [...] , [SolrDocument{id=12214}, SolrDocument{id=62269}, SolrDocument{id=12215}, SolrDocument{id=62270}, SolrDocument{id=12216}]
> [junit4:junit4] 2> ######cloudDocList: SolrDocumentList[sz=2357]=[SolrDocument{id=60002}, SolrDocument{id=10002}, SolrDocument{id=10007}, SolrDocument{id=60007}, SolrDocument{id=60008}] , [...] , [SolrDocument{id=12209}, SolrDocument{id=12210}, SolrDocument{id=12211}, SolrDocument{id=62269}, SolrDocument{id=12215}]
> [junit4:junit4] 2> ###### sizes=2356,2357
> [junit4:junit4] 2> ###### Only in cloudDocList: [{id=60140}]
> [junit4:junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.method=testDistribSearch -Dtests.seed=B7037FE5FE461C1 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar_AE -Dtests.timezone=Etc/GMT-4 -Dtests.file.encoding=US-ASCII
> [junit4:junit4] FAILURE 277s J1 | ChaosMonkeySafeLeaderTest.testDistribSearch <<<
> [junit4:junit4] > Throwable #1: java.lang.AssertionError: document count mismatch. control=2356 sum(shards)=2357 cloudClient=2357
> [junit4:junit4] > at __randomizedtesting.SeedInfo.seed([B7037FE5FE461C1:8A96B9E628BB01FD]:0)
> [junit4:junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1052)
> [junit4:junit4] > at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:137)
> [junit4:junit4] > at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:806)
> [junit4:junit4] > at java.lang.Thread.run(Thread.java:722)
> [junit4:junit4] 1> INFO - 2013-04-01 21:26:13.794; org.apache.solr.SolrTestCaseJ4; ###deleteCore
> [junit4:junit4] 2> 277347 T1680 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 2 leaked thread(s).
> [junit4:junit4] 1> INFO - 2013-04-01 21:26:14.001; org.apache.zookeeper.server.SessionTrackerImpl; SessionTrackerImpl exited loop!
> [junit4:junit4] 1> WARN - 2013-04-01 21:26:14.963; org.apache.solr.common.cloud.ZkStateReader$3; ZooKeeper watch triggered, but Solr cannot talk to ZK
> [junit4:junit4] 1> INFO - 2013-04-01 21:26:14.963; org.apache.solr.common.cloud.ZkStateReader$2; A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
> [junit4:junit4] 1> WARN - 2013-04-01 21:26:14.963; org.apache.solr.common.cloud.ZkStateReader$2; ZooKeeper watch triggered, but Solr cannot talk to ZK
> [junit4:junit4] 1> INFO - 2013-04-01 21:26:14.964; org.apache.zookeeper.ClientCnxn$EventThread; EventThread shut down
> [junit4:junit4] 2> NOTE: test params are: codec=SimpleText, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=ar_AE, timezone=Etc/GMT-4
> [junit4:junit4] 2> NOTE: Linux 3.2.0-39-generic i386/Oracle Corporation 1.7.0_17 (32-bit)/cpus=8,threads=2,free=88868056,total=329252864
> [junit4:junit4] 2> NOTE: All tests run in this JVM: [ZkSolrClientTest, SolrCmdDistributorTest, TestMergePolicyConfig, ZkNodePropsTest, TestBinaryResponseWriter, EchoParamsTest, TestSuggestSpellingConverter, TestJmxIntegration, RequiredFieldsTest, UniqFieldsUpdateProcessorFactoryTest, SolrCoreCheckLockOnStartupTest, DOMUtilTest, TestNumberUtils, TestSolrDiscoveryProperties, TestSchemaNameResource, TestIndexSearcher, TestZkChroot, SpellCheckCollatorTest, SimplePostToolTest, TestMultiCoreConfBootstrap, XmlUpdateRequestHandlerTest, TestSolrQueryParserDefaultOperatorResource, TestRTGBase, TestBM25SimilarityFactory, TestUpdate, BadCopyFieldTest, TestCollationField, TestStressLucene, NoCacheHeaderTest, PathHierarchyTokenizerFactoryTest, AliasIntegrationTest, TestCloudManagedSchema, TestUniqueKeyFieldResource, SolrRequestParserTest, TestFiltering, DistanceFunctionTest, IndexBasedSpellCheckerTest, UpdateParamsTest, FieldAnalysisRequestHandlerTest, TestAnalyzedSuggestions, RequestHandlersTest, OpenCloseCoreStressTest, TestBinaryField, ShardRoutingCustomTest, TestCSVLoader, TestQuerySenderNoQuery, TestIndexingPerformance, HighlighterTest, SpellCheckComponentTest, BasicDistributedZk2Test, ChaosMonkeySafeLeaderTest]
> [junit4:junit4] Completed on J1 in 278.62s, 1 test, 1 failure <<< FAILURES!
>
> [...truncated 488 lines...]
> BUILD FAILED
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:375: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:355: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:39: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:183: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:448: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1231: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:875: There were test failures: 284 suites, 1173 tests, 1 failure, 13 ignored (7 assumptions)
>
> Total time: 46 minutes 54 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Recording test results
> Description set: Java: 32bit/jdk1.7.0_17 -server -XX:+UseG1GC
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org