You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/03/14 21:55:53 UTC

[JENKINS] Lucene-Solr-4.x-Linux (32bit/jdk1.7.0_60-ea-b07) - Build # 9687 - Failure!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/9687/
Java: 32bit/jdk1.7.0_60-ea-b07 -client -XX:+UseConcMarkSweepGC

1 tests failed.
REGRESSION:  org.apache.solr.client.solrj.impl.CloudSolrServerTest.testShutdown

Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:37392 within 45000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:37392 within 45000 ms
	at __randomizedtesting.SeedInfo.seed([6FA1F7EC2B093C34:8CD7FE79CC73A946]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:150)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:101)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:91)
	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:89)
	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:83)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.setUp(AbstractDistribZkTestBase.java:70)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.setUp(AbstractFullDistribZkTestBase.java:201)
	at org.apache.solr.client.solrj.impl.CloudSolrServerTest.setUp(CloudSolrServerTest.java:78)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1617)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:860)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:876)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:359)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:783)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:443)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:835)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:771)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:782)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:359)
	at java.lang.Thread.run(Thread.java:744)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:37392 within 45000 ms
	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:223)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:142)
	... 45 more




Build Log:
[...truncated 11728 lines...]
   [junit4] Suite: org.apache.solr.client.solrj.impl.CloudSolrServerTest
   [junit4]   2> 26033 T67 oas.SolrTestCaseJ4.startTrackingSearchers WARN startTrackingSearchers: numOpens=6 numCloses=6
   [junit4]   2> 26033 T67 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 26034 T67 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 26035 T67 oasc.AbstractZkTestCase.<clinit> WARN TEST_HOME() does not exist - solrj test?
   [junit4]   2> 26038 T67 oas.SolrTestCaseJ4.setUp ###Starting testShutdown
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./solrtest-CloudSolrServerTest-1394830442074
   [junit4]   2> 26040 T67 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 26044 T68 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 26143 T67 oasc.ZkTestServer.run start zk server on port:37392
   [junit4]   2> 26182 T67 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 75008 T71 oazsp.FileTxnLog.commit WARN fsync-ing the write ahead log in SyncThread:0 took 48813ms which will adversely effect operation latency. See the ZooKeeper troubleshooting guide
   [junit4]   2> 75010 T67 oas.SolrTestCaseJ4.tearDown ###Ending testShutdown
   [junit4]   2> 75036 T69 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x144c25f7e9e0000, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 75037 T67 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:37392 37392
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CloudSolrServerTest -Dtests.method=testShutdown -Dtests.seed=6FA1F7EC2B093C34 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=is -Dtests.timezone=Africa/Luanda -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   49.2s J0 | CloudSolrServerTest.testShutdown <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:37392 within 45000 ms
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([6FA1F7EC2B093C34:8CD7FE79CC73A946]:0)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:150)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:101)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:91)
   [junit4]    > 	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:89)
   [junit4]    > 	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:83)
   [junit4]    > 	at org.apache.solr.cloud.AbstractDistribZkTestBase.setUp(AbstractDistribZkTestBase.java:70)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.setUp(AbstractFullDistribZkTestBase.java:201)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServerTest.setUp(CloudSolrServerTest.java:78)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:744)
   [junit4]    > Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:37392 within 45000 ms
   [junit4]    > 	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:223)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:142)
   [junit4]    > 	... 45 more
   [junit4]   2> 75256 T67 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./solrtest-CloudSolrServerTest-1394830491291
   [junit4]   2> 75258 T67 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 75259 T75 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 75359 T67 oasc.ZkTestServer.run start zk server on port:48577
   [junit4]   2> 75360 T67 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 75368 T81 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5e4dd0 name:ZooKeeperConnection Watcher:127.0.0.1:48577 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 75369 T67 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 75370 T67 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 75395 T67 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 75397 T83 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9bfe43 name:ZooKeeperConnection Watcher:127.0.0.1:48577/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 75397 T67 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 75401 T67 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 75408 T67 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 75411 T67 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 75413 T67 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 75417 T67 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 75424 T67 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 75432 T67 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 75432 T67 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 75437 T67 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml because it doesn't exist
   [junit4]   2> 75437 T67 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/stopwords.txt because it doesn't exist
   [junit4]   2> 75438 T67 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/protwords.txt because it doesn't exist
   [junit4]   2> 75438 T67 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/currency.xml because it doesn't exist
   [junit4]   2> 75438 T67 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/enumsConfig.xml because it doesn't exist
   [junit4]   2> 75439 T67 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/open-exchange-rates.json because it doesn't exist
   [junit4]   2> 75439 T67 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/mapping-ISOLatin1Accent.txt because it doesn't exist
   [junit4]   2> 75439 T67 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/old_synonyms.txt because it doesn't exist
   [junit4]   2> 75440 T67 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/synonyms.txt because it doesn't exist
   [junit4]   2> 75444 T67 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 75445 T85 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@17c4faf name:ZooKeeperConnection Watcher:127.0.0.1:48577/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 75446 T67 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 75514 T67 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 75629 T67 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 75632 T67 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:54961
   [junit4]   2> 75637 T67 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 75637 T67 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 75638 T67 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1394830491483
   [junit4]   2> 75638 T67 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1394830491483/'
   [junit4]   2> 75665 T67 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1394830491483/solr.xml
   [junit4]   2> 75716 T67 oasc.CoreContainer.<init> New CoreContainer 19504566
   [junit4]   2> 75717 T67 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1394830491483/]
   [junit4]   2> 75734 T67 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 75735 T67 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 75735 T67 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 75736 T67 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 75736 T67 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 75736 T67 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 75736 T67 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 75737 T67 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 75737 T67 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 75746 T67 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 75747 T67 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 75747 T67 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 75747 T67 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48577/solr
   [junit4]   2> 75756 T67 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 75757 T67 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 75760 T96 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c0bb57 name:ZooKeeperConnection Watcher:127.0.0.1:48577 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 75761 T67 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 75768 T67 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 75770 T98 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74a9a9 name:ZooKeeperConnection Watcher:127.0.0.1:48577/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 75770 T67 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 75779 T67 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 75787 T67 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 75805 T67 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 75809 T67 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54961_
   [junit4]   2> 75813 T67 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54961_
   [junit4]   2> 75818 T67 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 75821 T67 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 75831 T67 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:54961_
   [junit4]   2> 75831 T67 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 75836 T67 oasc.Overseer.start Overseer (id=91411611077640196-127.0.0.1:54961_-n_0000000000) starting
   [junit4]   2> 75923 T67 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 75936 T100 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 75937 T67 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 75938 T100 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 75942 T67 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 75944 T67 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 75951 T99 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 75957 T101 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 75958 T101 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 75960 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 75961 T101 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 75963 T99 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 75964 T99 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:54961",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54961_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 75965 T99 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 75974 T99 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 75978 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 75982 T98 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 76961 T101 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 76961 T101 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1394830491483/collection1
   [junit4]   2> 76962 T101 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 76963 T101 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 76963 T101 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 76964 T101 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 76965 T101 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1394830491483/collection1/'
   [junit4]   2> 77007 T101 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 77048 T101 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 77050 T101 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 77069 T101 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 77121 T101 oasc.SolrResourceLoader.findClass WARN Solr loaded a deprecated plugin/analysis class [solr.SortableIntField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 77129 T101 oasc.SolrResourceLoader.findClass WARN Solr loaded a deprecated plugin/analysis class [solr.SortableLongField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 77134 T101 oasc.SolrResourceLoader.findClass WARN Solr loaded a deprecated plugin/analysis class [solr.SortableFloatField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 77140 T101 oasc.SolrResourceLoader.findClass WARN Solr loaded a deprecated plugin/analysis class [solr.SortableDoubleField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 77566 T101 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 77569 T101 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 77571 T101 oass.IndexSchema.readSchema WARN Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 77571 T101 oass.IndexSchema.readSchema WARN Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 77574 T101 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 77574 T101 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1394830491483/collection1/, dataDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/control/data/
   [junit4]   2> 77574 T101 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 77575 T101 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/control/data
   [junit4]   2> 77575 T101 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/control/data/index/
   [junit4]   2> 77575 T101 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 77576 T101 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/control/data/index
   [junit4]   2> 77577 T101 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1e74739 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/control/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 77577 T101 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 77578 T101 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 77579 T101 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 77579 T101 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 77579 T101 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 77579 T101 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 77579 T101 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 77580 T101 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 77580 T101 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: solr.PingRequestHandler
   [junit4]   2> 77582 T101 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 77584 T101 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 77584 T101 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 77585 T101 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1e74739 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/control/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 77586 T101 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 77586 T101 oass.SolrIndexSearcher.<init> Opening Searcher@92c29d[collection1] main
   [junit4]   2> 77587 T102 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@92c29d[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 77587 T101 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 77588 T67 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0
   [junit4]   2> 77588 T67 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 77588 T105 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:54961 collection:control_collection shard:shard1
   [junit4]   2> 77605 T67 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 77608 T105 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 77608 T107 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18ed660 name:ZooKeeperConnection Watcher:127.0.0.1:48577/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 77608 T67 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 77610 T67 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 77617 T105 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 77619 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77620 T105 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 77620 T105 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 77621 T99 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 77621 T67 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> ASYNC  NEW_CORE C63 name=collection1 org.apache.solr.core.SolrCore@1690a61 url=https://127.0.0.1:54961/collection1 node=127.0.0.1:54961_ C63_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:54961, core=collection1, node_name=127.0.0.1:54961_}
   [junit4]   2> 77621 T105 C63 P54961 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:54961/collection1/
   [junit4]   2> 77622 T105 C63 P54961 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 77622 T105 C63 P54961 oasc.SyncStrategy.syncToMe https://127.0.0.1:54961/collection1/ has no replicas
   [junit4]   2> 77623 T105 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:54961/collection1/ shard1
   [junit4]   2> 77624 T105 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 77628 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77629 T107 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 77629 T98 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 77657 T67 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 77658 T67 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 77662 T67 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 77664 T67 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:50906
   [junit4]   2> 77673 T67 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 77673 T67 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 77674 T67 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1394830493657
   [junit4]   2> 77674 T67 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1394830493657/'
   [junit4]   2> 77692 T67 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1394830493657/solr.xml
   [junit4]   2> 77746 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77748 T99 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 77751 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77760 T67 oasc.CoreContainer.<init> New CoreContainer 24599131
   [junit4]   2> 77760 T67 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1394830493657/]
   [junit4]   2> 77761 T67 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 77762 T67 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 77762 T67 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 77763 T67 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 77763 T67 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 77763 T67 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 77764 T67 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 77764 T67 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 77765 T67 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 77771 T67 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 77772 T67 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 77772 T67 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 77773 T67 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48577/solr
   [junit4]   2> 77773 T67 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 77774 T67 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 77776 T118 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1db5913 name:ZooKeeperConnection Watcher:127.0.0.1:48577 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 77776 T67 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 77779 T67 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 77781 T120 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b0b267 name:ZooKeeperConnection Watcher:127.0.0.1:48577/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 77781 T67 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 77786 T67 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 77854 T120 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 77854 T98 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 77854 T107 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 77897 T105 oasc.ZkController.register We are https://127.0.0.1:54961/collection1/ and leader is https://127.0.0.1:54961/collection1/
   [junit4]   2> 77897 T105 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:54961
   [junit4]   2> 77897 T105 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 77898 T105 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 77898 T105 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 77900 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77900 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77900 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77902 T105 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 77904 T99 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 77905 T99 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:54961",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54961_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 77909 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 78013 T98 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 78014 T120 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 78014 T107 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 78791 T67 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50906_
   [junit4]   2> 78793 T67 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50906_
   [junit4]   2> 78796 T107 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 78796 T120 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 78796 T98 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 78805 T121 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 78805 T121 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 78806 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 78807 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 78807 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 78807 T121 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 78808 T99 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 78809 T99 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:50906",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50906_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 78810 T99 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 78810 T99 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 78820 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 78922 T107 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 78922 T98 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 78922 T120 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 79807 T121 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 79808 T121 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1394830493657/collection1
   [junit4]   2> 79808 T121 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 79809 T121 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 79809 T121 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 79810 T121 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 79810 T121 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1394830493657/collection1/'
   [junit4]   2> 79856 T121 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 79904 T121 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 79906 T121 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 79924 T121 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 80347 T121 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 80352 T121 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 80353 T121 oass.IndexSchema.readSchema WARN Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 80354 T121 oass.IndexSchema.readSchema WARN Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 80354 T121 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 80355 T121 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1394830493657/collection1/, dataDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty1/
   [junit4]   2> 80355 T121 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 80356 T121 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty1
   [junit4]   2> 80356 T121 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty1/index/
   [junit4]   2> 80356 T121 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 80356 T121 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty1/index
   [junit4]   2> 80357 T121 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1e2b197 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty1/index),segFN=segments_1,generation=1}
   [junit4]   2> 80358 T121 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 80359 T121 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 80359 T121 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 80359 T121 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 80359 T121 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 80359 T121 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 80360 T121 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 80360 T121 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 80360 T121 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: solr.PingRequestHandler
   [junit4]   2> 80362 T121 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 80368 T121 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 80369 T121 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 80370 T121 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1e2b197 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty1/index),segFN=segments_1,generation=1}
   [junit4]   2> 80370 T121 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 80370 T121 oass.SolrIndexSearcher.<init> Opening Searcher@1cc66d1[collection1] main
   [junit4]   2> 80371 T122 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1cc66d1[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 80372 T121 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 80373 T67 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0
   [junit4]   2> 80373 T67 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 80373 T125 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:50906 collection:collection1 shard:shard2
   [junit4]   2> 80375 T125 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 80384 T125 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 80389 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80389 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80389 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80390 T125 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 80390 T125 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C64 name=collection1 org.apache.solr.core.SolrCore@9b6b8c url=https://127.0.0.1:50906/collection1 node=127.0.0.1:50906_ C64_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:50906, core=collection1, node_name=127.0.0.1:50906_}
   [junit4]   2> 80390 T125 C64 P50906 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:50906/collection1/
   [junit4]   2> 80391 T125 C64 P50906 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 80391 T99 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 80391 T125 C64 P50906 oasc.SyncStrategy.syncToMe https://127.0.0.1:50906/collection1/ has no replicas
   [junit4]   2> 80391 T125 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:50906/collection1/ shard2
   [junit4]   2> 80391 T125 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 80395 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80401 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80404 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80412 T67 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 80412 T67 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 80416 T67 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 80419 T67 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:56122
   [junit4]   2> 80421 T67 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 80422 T67 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 80422 T67 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1394830496410
   [junit4]   2> 80422 T67 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1394830496410/'
   [junit4]   2> 80441 T67 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1394830496410/solr.xml
   [junit4]   2> 80500 T67 oasc.CoreContainer.<init> New CoreContainer 29234952
   [junit4]   2> 80501 T67 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1394830496410/]
   [junit4]   2> 80501 T67 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 80502 T67 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 80502 T67 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 80502 T67 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 80502 T67 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 80503 T67 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 80503 T67 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 80503 T67 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 80504 T67 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 80509 T98 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 80509 T120 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 80509 T67 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 80509 T107 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 80510 T67 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 80510 T67 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 80511 T67 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48577/solr
   [junit4]   2> 80511 T67 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 80512 T67 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 80514 T136 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@140500e name:ZooKeeperConnection Watcher:127.0.0.1:48577 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 80514 T67 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 80517 T67 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 80518 T138 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1370ce8 name:ZooKeeperConnection Watcher:127.0.0.1:48577/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 80518 T67 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 80523 T67 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 80551 T125 oasc.ZkController.register We are https://127.0.0.1:50906/collection1/ and leader is https://127.0.0.1:50906/collection1/
   [junit4]   2> 80552 T125 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:50906
   [junit4]   2> 80552 T125 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 80552 T125 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 80552 T125 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 80554 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80554 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80554 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80554 T125 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 80566 T99 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 80567 T99 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:50906",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50906_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 80570 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80672 T98 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 80672 T120 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 80673 T138 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 80673 T107 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 81527 T67 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56122_
   [junit4]   2> 81528 T67 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:56122_
   [junit4]   2> 81531 T120 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 81531 T138 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 81531 T98 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 81531 T107 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 81539 T139 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 81540 T139 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 81541 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 81541 T139 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 81542 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 81542 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 81543 T99 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 81544 T99 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:56122",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:56122_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 81544 T99 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 81545 T99 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 81548 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 81650 T120 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 81650 T98 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 81650 T107 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 81650 T138 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 82542 T139 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 82542 T139 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1394830496410/collection1
   [junit4]   2> 82542 T139 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 82543 T139 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 82544 T139 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 82545 T139 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 82545 T139 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1394830496410/collection1/'
   [junit4]   2> 82585 T139 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 82622 T139 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 82623 T139 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 82633 T139 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 83004 T139 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 83006 T139 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 83008 T139 oass.IndexSchema.readSchema WARN Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 83008 T139 oass.IndexSchema.readSchema WARN Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 83009 T139 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 83009 T139 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1394830496410/collection1/, dataDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty2/
   [junit4]   2> 83009 T139 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 83010 T139 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty2
   [junit4]   2> 83010 T139 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty2/index/
   [junit4]   2> 83010 T139 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 83011 T139 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty2/index
   [junit4]   2> 83012 T139 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@13559a9 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty2/index),segFN=segments_1,generation=1}
   [junit4]   2> 83012 T139 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 83013 T139 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 83013 T139 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 83014 T139 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 83014 T139 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 83014 T139 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 83014 T139 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 83014 T139 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 83015 T139 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: solr.PingRequestHandler
   [junit4]   2> 83017 T139 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 83022 T139 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 83022 T139 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 83023 T139 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@13559a9 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty2/index),segFN=segments_1,generation=1}
   [junit4]   2> 83023 T139 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 83024 T139 oass.SolrIndexSearcher.<init> Opening Searcher@5586d0[collection1] main
   [junit4]   2> 83025 T140 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5586d0[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 83025 T139 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 83026 T67 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0
   [junit4]   2> 83026 T67 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 83026 T143 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:56122 collection:collection1 shard:shard1
   [junit4]   2> 83028 T143 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 83034 T143 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 83036 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 83036 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 83036 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 83040 T143 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 83040 T99 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 83040 T143 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C65 name=collection1 org.apache.solr.core.SolrCore@f89046 url=https://127.0.0.1:56122/collection1 node=127.0.0.1:56122_ C65_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:56122, core=collection1, node_name=127.0.0.1:56122_}
   [junit4]   2> 83040 T143 C65 P56122 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:56122/collection1/
   [junit4]   2> 83041 T143 C65 P56122 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 83041 T143 C65 P56122 oasc.SyncStrategy.syncToMe https://127.0.0.1:56122/collection1/ has no replicas
   [junit4]   2> 83041 T143 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:56122/collection1/ shard1
   [junit4]   2> 83041 T143 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 83044 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 83048 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 83053 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 83062 T67 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 83063 T67 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 83066 T67 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 83072 T67 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:52678
   [junit4]   2> 83074 T67 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 83074 T67 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 83074 T67 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1394830499063
   [junit4]   2> 83074 T67 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1394830499063/'
   [junit4]   2> 83096 T67 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1394830499063/solr.xml
   [junit4]   2> 83149 T67 oasc.CoreContainer.<init> New CoreContainer 30164446
   [junit4]   2> 83149 T67 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1394830499063/]
   [junit4]   2> 83150 T67 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 83151 T67 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 83151 T67 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 83151 T67 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 83152 T67 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 83152 T67 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 83152 T67 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 83153 T67 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 83153 T67 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 83160 T67 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 83160 T67 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 83160 T98 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 83161 T107 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 83161 T138 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 83161 T67 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 83160 T120 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 83162 T67 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48577/solr
   [junit4]   2> 83164 T67 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 83164 T67 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 83171 T154 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18458dd name:ZooKeeperConnection Watcher:127.0.0.1:48577 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 83171 T67 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 83173 T67 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 83174 T156 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1854bbf name:ZooKeeperConnection Watcher:127.0.0.1:48577/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 83175 T67 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 83179 T67 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 83200 T143 oasc.ZkController.register We are https://127.0.0.1:56122/collection1/ and leader is https://127.0.0.1:56122/collection1/
   [junit4]   2> 83200 T143 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:56122
   [junit4]   2> 83201 T143 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 83201 T143 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 83201 T143 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 83202 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 83202 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 83202 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 83203 T143 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 83204 T99 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 83205 T99 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:56122",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:56122_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 83207 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 83309 T98 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 83310 T156 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 83310 T120 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 83309 T107 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 83309 T138 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 84182 T67 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52678_
   [junit4]   2> 84183 T67 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:52678_
   [junit4]   2> 84187 T107 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 84187 T120 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 84187 T98 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 84187 T156 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 84187 T138 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 84195 T157 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 84196 T157 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 84197 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 84197 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 84197 T157 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 84197 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 84198 T99 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 84199 T99 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:52678",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52678_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 84200 T99 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 84200 T99 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 84203 T98 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 84306 T98 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 84306 T138 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 84306 T120 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 84306 T156 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 84306 T107 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 85197 T157 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 85198 T157 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1394830499063/collection1
   [junit4]   2> 85198 T157 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 85199 T157 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 85199 T157 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 85200 T157 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 85201 T157 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1394830499063/collection1/'
   [junit4]   2> 85240 T157 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 85276 T157 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 85278 T157 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 85290 T157 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 85678 T157 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 85683 T157 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 85684 T157 oass.IndexSchema.readSchema WARN Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 85685 T157 oass.IndexSchema.readSchema WARN Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 85686 T157 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 85686 T157 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1394830499063/collection1/, dataDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty3/
   [junit4]   2> 85686 T157 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 85687 T157 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty3
   [junit4]   2> 85687 T157 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty3/index/
   [junit4]   2> 85688 T157 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty3/index' doesn't exist. Creating new index...
   [junit4]   2> 85688 T157 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty3/index
   [junit4]   2> 85690 T157 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@16d74ec lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty3/index),segFN=segments_1,generation=1}
   [junit4]   2> 85690 T157 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 85691 T157 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 85692 T157 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 85692 T157 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 85692 T157 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 85692 T157 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 85693 T157 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 85693 T157 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 85693 T157 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: solr.PingRequestHandler
   [junit4]   2> 85695 T157 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 85697 T157 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 85697 T157 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 85698 T157 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@16d74ec lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty3/index),segFN=segments_1,generation=1}
   [junit4]   2> 85699 T157 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 85699 T157 oass.SolrIndexSearcher.<init> Opening Searcher@9c5e02[collection1] main
   [junit4]   2> 85700 T158 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@9c5e02[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 85700 T157 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 85701 T67 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0
   [junit4]   2> 85702 T67 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 85701 T161 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:52678 collection:collection1 shard:shard2
   [junit4]   2> 85706 T161 oasc.ZkController.register We are https://127.0.0.1:52678/collection1/ and leader is https://127.0.0.1:50906/collection1/
   [junit4]   2> 85706 T161 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:52678
   [junit4]   2> 85706 T161 oasc.ZkController.checkRecovery Core needs to recover:collection1
   [junit4]   2> 85706 T161 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
   [junit4]   2> ASYNC  NEW_CORE C66 name=collection1 org.apache.solr.core.SolrCore@1ee63b7 url=https://127.0.0.1:52678/collection1 node=127.0.0.1:52678_ C66_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:52678, core=collection1, node_name=127.0.0.1:52678_}
   [junit4]   2> 85710 T162 C66 P52678 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
   [junit4]   2> 85711 T161 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 85711 T162 C66 P52678 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
   [junit4]   2> 85712 T162 C66 P52678 oasc.ZkController.publish publishing core=collection1 state=recovering collection=collection1
   [junit4]   2> 85712 T162 C66 P52678 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 85713 T98 oasc.DistributedQueue$LatchChildWa

[...truncated too long message...]

lose closing DirectUpdateHandler2{commits=6,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=2,cumulative_deletesById=1,cumulative_deletesByQuery=2,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=5}
   [junit4]   2> 98776 T156 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 98777 T156 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 98777 T156 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 98779 T156 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 98779 T156 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 98780 T156 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty3 [CachedDir<<refCount=0;path=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty3;done=false>>]
   [junit4]   2> 98780 T156 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty3
   [junit4]   2> 98780 T156 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty3/index [CachedDir<<refCount=0;path=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty3/index;done=false>>]
   [junit4]   2> 98781 T156 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty3/index
   [junit4]   2> 98781 T156 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 98781 T156 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 98782 T156 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:259)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 98783 T156 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> ASYNC  NEW_CORE C117 name=collection1 org.apache.solr.core.SolrCore@18c9791 url=https://127.0.0.1:46975/collection1 node=127.0.0.1:46975_ C117_STATE=coll:collection1 core:collection1 props:{state=active, base_url=https://127.0.0.1:46975, core=collection1, node_name=127.0.0.1:46975_}
   [junit4]   2> 98857 T175 C117 P46975 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
   [junit4]   2> 98858 T175 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is shutdown
   [junit4]   2> 98858 T175 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@18c9791
   [junit4]   2> 98859 T175 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=6,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=2,cumulative_deletesById=1,cumulative_deletesByQuery=2,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=5}
   [junit4]   2> 98859 T175 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 98859 T175 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 98859 T175 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 98861 T175 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 98861 T175 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 98861 T175 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty4 [CachedDir<<refCount=0;path=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty4;done=false>>]
   [junit4]   2> 98861 T175 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty4
   [junit4]   2> 98861 T175 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty4/index [CachedDir<<refCount=0;path=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty4/index;done=false>>]
   [junit4]   2> 98862 T175 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1394830491291/jetty4/index
   [junit4]   2> 98862 T175 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:259)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 98862 T175 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: test params are: codec=Lucene46: {range_facet_si=PostingsFormat(name=TestBloomFilteredLucene41Postings), multiDefault=PostingsFormat(name=TestBloomFilteredLucene41Postings), _version_=Lucene41(blocksize=128), text=PostingsFormat(name=Lucene41WithOrds), range_facet_sl=Pulsing41(freqCutoff=20 minBlockSize=58 maxBlockSize=206), id=Pulsing41(freqCutoff=20 minBlockSize=58 maxBlockSize=206), intDefault=Lucene41(blocksize=128), timestamp=PostingsFormat(name=TestBloomFilteredLucene41Postings), a_t=PostingsFormat(name=TestBloomFilteredLucene41Postings), range_facet_l=PostingsFormat(name=TestBloomFilteredLucene41Postings)}, docValues:{}, sim=DefaultSimilarity, locale=is, timezone=Africa/Luanda
   [junit4]   2> NOTE: Linux 3.8.0-36-generic i386/Oracle Corporation 1.7.0_60-ea (32-bit)/cpus=8,threads=1,free=5824496,total=36761600
   [junit4]   2> NOTE: All tests run in this JVM: [SolrExampleStreamingBinaryTest, TestRetryUtil, TestUpdateRequestCodec, DocumentAnalysisResponseTest, TestEmbeddedSolrServer, CloudSolrServerTest]
   [junit4] Completed on J0 in 73.82s, 2 tests, 1 error <<< FAILURES!

[...truncated 103 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:202: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:490: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1276: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:908: There were test failures: 49 suites, 283 tests, 1 error

Total time: 55 minutes 11 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.7.0_60-ea-b07 -client -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure