You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/02/26 05:36:31 UTC

[JENKINS] Lucene-Solr-4.x-Linux (32bit/jdk1.7.0_60-ea-b04) - Build # 9491 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/9491/
Java: 32bit/jdk1.7.0_60-ea-b04 -client -XX:+UseParallelGC

1 tests failed.
FAILED:  org.apache.solr.client.solrj.impl.CloudSolrServerTest.testShutdown

Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:43685 within 30000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:43685 within 30000 ms
	at __randomizedtesting.SeedInfo.seed([F2EA2531445F5726:119C2CA4A325C254]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:148)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:94)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:85)
	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:89)
	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:83)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.setUp(AbstractDistribZkTestBase.java:70)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.setUp(AbstractFullDistribZkTestBase.java:200)
	at org.apache.solr.client.solrj.impl.CloudSolrServerTest.setUp(CloudSolrServerTest.java:77)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:771)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:744)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:43685 within 30000 ms
	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:222)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:140)
	... 47 more




Build Log:
[...truncated 11539 lines...]
   [junit4] Suite: org.apache.solr.client.solrj.impl.CloudSolrServerTest
   [junit4]   2> 5832 T40 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 5833 T40 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /f_/y
   [junit4]   2> 5834 T40 oasc.AbstractZkTestCase.<clinit> WARN TEST_HOME() does not exist - solrj test?
   [junit4]   2> 5838 T40 oas.SolrTestCaseJ4.setUp ###Starting testShutdown
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./solrtest-CloudSolrServerTest-1393389252072
   [junit4]   2> 5840 T40 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 5842 T41 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 5942 T40 oasc.ZkTestServer.run start zk server on port:43685
   [junit4]   2> 5973 T40 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 67921 T44 oazsp.FileTxnLog.commit WARN fsync-ing the write ahead log in SyncThread:0 took 61936ms which will adversely effect operation latency. See the ZooKeeper troubleshooting guide
   [junit4]   2> 67922 T40 oas.SolrTestCaseJ4.tearDown ###Ending testShutdown
   [junit4]   2> 67930 T42 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1446c78ae1a0000, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 67931 T40 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:43685 43685
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CloudSolrServerTest -Dtests.method=testShutdown -Dtests.seed=F2EA2531445F5726 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=bg_BG -Dtests.timezone=Zulu -Dtests.file.encoding=UTF-8
   [junit4] ERROR   67.4s J0 | CloudSolrServerTest.testShutdown <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:43685 within 30000 ms
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F2EA2531445F5726:119C2CA4A325C254]:0)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:148)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:94)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:85)
   [junit4]    > 	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:89)
   [junit4]    > 	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:83)
   [junit4]    > 	at org.apache.solr.cloud.AbstractDistribZkTestBase.setUp(AbstractDistribZkTestBase.java:70)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.setUp(AbstractFullDistribZkTestBase.java:200)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServerTest.setUp(CloudSolrServerTest.java:77)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:744)
   [junit4]    > Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:43685 within 30000 ms
   [junit4]    > 	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:222)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:140)
   [junit4]    > 	... 47 more
   [junit4]   2> 73228 T40 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./solrtest-CloudSolrServerTest-1393389319462
   [junit4]   2> 73228 T40 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 73229 T48 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 73329 T40 oasc.ZkTestServer.run start zk server on port:60544
   [junit4]   2> 73330 T40 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 73423 T54 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c92b5d name:ZooKeeperConnection Watcher:127.0.0.1:60544 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 73423 T40 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 73424 T40 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 73442 T40 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 73443 T56 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b2a524 name:ZooKeeperConnection Watcher:127.0.0.1:60544/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 73444 T40 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 73447 T40 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 73452 T40 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 73455 T40 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 73458 T40 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 73462 T40 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 73468 T40 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 73473 T40 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 73474 T40 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 73478 T40 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml because it doesn't exist
   [junit4]   2> 73478 T40 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/stopwords.txt because it doesn't exist
   [junit4]   2> 73478 T40 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/protwords.txt because it doesn't exist
   [junit4]   2> 73479 T40 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/currency.xml because it doesn't exist
   [junit4]   2> 73479 T40 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/enumsConfig.xml because it doesn't exist
   [junit4]   2> 73479 T40 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/open-exchange-rates.json because it doesn't exist
   [junit4]   2> 73480 T40 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/mapping-ISOLatin1Accent.txt because it doesn't exist
   [junit4]   2> 73480 T40 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/old_synonyms.txt because it doesn't exist
   [junit4]   2> 73480 T40 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/synonyms.txt because it doesn't exist
   [junit4]   2> 73485 T40 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 73487 T58 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b8ad39 name:ZooKeeperConnection Watcher:127.0.0.1:60544/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 73487 T40 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 73527 T40 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 73531 T40 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 73534 T40 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:45574
   [junit4]   2> 73535 T40 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 73536 T40 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 73536 T40 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1393389319724
   [junit4]   2> 73536 T40 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1393389319724/'
   [junit4]   2> 73554 T40 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1393389319724/solr.xml
   [junit4]   2> 73581 T40 oasc.CoreContainer.<init> New CoreContainer 5554378
   [junit4]   2> 73582 T40 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1393389319724/]
   [junit4]   2> 73582 T40 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 73583 T40 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 73583 T40 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 73584 T40 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 73584 T40 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 73584 T40 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 73584 T40 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 73585 T40 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 73585 T40 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 73590 T40 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 73590 T40 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 73590 T40 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 73591 T40 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:60544/solr
   [junit4]   2> 73599 T40 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 73600 T40 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 73601 T69 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1bdc261 name:ZooKeeperConnection Watcher:127.0.0.1:60544 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 73602 T40 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 73607 T40 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 73608 T71 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@37ea3c name:ZooKeeperConnection Watcher:127.0.0.1:60544/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 73608 T40 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 73613 T40 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 73618 T40 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 73630 T40 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 73632 T40 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45574_f_%2Fy
   [junit4]   2> 73634 T40 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45574_f_%2Fy
   [junit4]   2> 73638 T40 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 73640 T40 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 73647 T40 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:45574_f_%2Fy
   [junit4]   2> 73647 T40 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 73651 T40 oasc.Overseer.start Overseer (id=91317162440392708-127.0.0.1:45574_f_%2Fy-n_0000000000) starting
   [junit4]   2> 73662 T40 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 73674 T73 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 73675 T40 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 73676 T73 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 73678 T40 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 73680 T40 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 73686 T72 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 73692 T74 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 73693 T74 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 73695 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 73695 T74 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 73698 T72 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 73699 T72 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:45574/f_/y",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45574_f_%2Fy",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 73699 T72 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 73706 T72 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 73709 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 73712 T71 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 74696 T74 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 74696 T74 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1393389319724/collection1
   [junit4]   2> 74696 T74 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 74697 T74 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 74698 T74 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 74699 T74 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 74700 T74 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1393389319724/collection1/'
   [junit4]   2> 74731 T74 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 74754 T74 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 74756 T74 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 74764 T74 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 74792 T74 oasc.SolrResourceLoader.findClass WARN Solr loaded a deprecated plugin/analysis class [solr.SortableIntField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 74797 T74 oasc.SolrResourceLoader.findClass WARN Solr loaded a deprecated plugin/analysis class [solr.SortableLongField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 74802 T74 oasc.SolrResourceLoader.findClass WARN Solr loaded a deprecated plugin/analysis class [solr.SortableFloatField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 74807 T74 oasc.SolrResourceLoader.findClass WARN Solr loaded a deprecated plugin/analysis class [solr.SortableDoubleField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 75049 T74 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 75051 T74 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 75053 T74 oass.IndexSchema.readSchema WARN Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 75053 T74 oass.IndexSchema.readSchema WARN Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 75056 T74 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 75056 T74 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1393389319724/collection1/, dataDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/control/data/
   [junit4]   2> 75056 T74 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 75057 T74 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/control/data
   [junit4]   2> 75057 T74 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/control/data/index/
   [junit4]   2> 75058 T74 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 75058 T74 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/control/data/index
   [junit4]   2> 75059 T74 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@17faa18 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/control/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 75059 T74 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 75060 T74 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 75061 T74 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 75061 T74 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 75061 T74 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 75065 T74 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 75065 T74 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 75065 T74 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 75066 T74 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: solr.PingRequestHandler
   [junit4]   2> 75068 T74 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 75070 T74 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 75070 T74 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 75071 T74 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@17faa18 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/control/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 75071 T74 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 75071 T74 oass.SolrIndexSearcher.<init> Opening Searcher@4266b6[collection1] main
   [junit4]   2> 75072 T75 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4266b6[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 75072 T74 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 75073 T40 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0
   [junit4]   2> 75074 T40 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 75073 T78 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:45574/f_/y collection:control_collection shard:shard1
   [junit4]   2> 75080 T78 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 75083 T40 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 75084 T80 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16a4a4f name:ZooKeeperConnection Watcher:127.0.0.1:60544/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 75084 T40 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 75087 T40 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 75089 T78 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 75092 T40 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 75099 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 75100 T78 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 75100 T78 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 75102 T72 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C3 name=collection1 org.apache.solr.core.SolrCore@c888a6 url=https://127.0.0.1:45574/f_/y/collection1 node=127.0.0.1:45574_f_%2Fy C3_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:45574/f_/y, core=collection1, node_name=127.0.0.1:45574_f_%2Fy}
   [junit4]   2> 75103 T78 C3 P45574 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:45574/f_/y/collection1/
   [junit4]   2> 75104 T78 C3 P45574 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 75104 T78 C3 P45574 oasc.SyncStrategy.syncToMe https://127.0.0.1:45574/f_/y/collection1/ has no replicas
   [junit4]   2> 75105 T78 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:45574/f_/y/collection1/ shard1
   [junit4]   2> 75106 T78 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 75107 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 75113 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 75132 T40 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 75133 T40 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 75135 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 75138 T40 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 75141 T40 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:38879
   [junit4]   2> 75142 T40 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 75143 T40 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 75143 T40 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1393389321326
   [junit4]   2> 75143 T40 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1393389321326/'
   [junit4]   2> 75155 T40 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1393389321326/solr.xml
   [junit4]   2> 75183 T40 oasc.CoreContainer.<init> New CoreContainer 1592961
   [junit4]   2> 75184 T40 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1393389321326/]
   [junit4]   2> 75185 T40 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 75185 T40 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 75185 T40 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 75186 T40 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 75186 T40 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 75186 T40 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 75187 T40 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 75187 T40 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 75187 T40 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 75192 T40 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 75193 T40 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 75193 T40 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 75193 T40 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:60544/solr
   [junit4]   2> 75194 T40 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 75197 T40 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 75201 T91 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14b20d3 name:ZooKeeperConnection Watcher:127.0.0.1:60544 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 75201 T40 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 75203 T40 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 75205 T93 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4ca3bd name:ZooKeeperConnection Watcher:127.0.0.1:60544/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 75205 T40 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 75211 T40 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 75238 T71 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 75238 T93 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 75238 T80 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 75265 T78 oasc.ZkController.register We are https://127.0.0.1:45574/f_/y/collection1/ and leader is https://127.0.0.1:45574/f_/y/collection1/
   [junit4]   2> 75265 T78 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:45574/f_/y
   [junit4]   2> 75265 T78 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 75266 T78 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 75266 T78 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 75267 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 75268 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 75268 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 75268 T78 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 75270 T72 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 75271 T72 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:45574/f_/y",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45574_f_%2Fy",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 75274 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 75377 T80 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 75377 T93 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 75377 T71 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 76217 T40 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38879_f_%2Fy
   [junit4]   2> 76219 T40 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38879_f_%2Fy
   [junit4]   2> 76222 T80 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 76222 T93 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 76222 T71 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 76230 T94 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 76231 T94 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 76232 T94 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 76232 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 76232 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 76233 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 76234 T72 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 76235 T72 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:38879/f_/y",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38879_f_%2Fy",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 76235 T72 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 76235 T72 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 76238 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 76342 T71 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 76342 T93 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 76342 T80 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 77232 T94 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 77233 T94 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1393389321326/collection1
   [junit4]   2> 77233 T94 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 77234 T94 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 77234 T94 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 77235 T94 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 77236 T94 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1393389321326/collection1/'
   [junit4]   2> 77262 T94 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 77289 T94 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 77291 T94 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 77299 T94 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 77520 T94 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 77522 T94 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 77523 T94 oass.IndexSchema.readSchema WARN Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 77524 T94 oass.IndexSchema.readSchema WARN Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 77524 T94 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 77524 T94 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1393389321326/collection1/, dataDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty1/
   [junit4]   2> 77525 T94 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 77525 T94 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty1
   [junit4]   2> 77525 T94 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty1/index/
   [junit4]   2> 77526 T94 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 77526 T94 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty1/index
   [junit4]   2> 77527 T94 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1718107 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty1/index),segFN=segments_1,generation=1}
   [junit4]   2> 77527 T94 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 77528 T94 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 77529 T94 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 77529 T94 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 77529 T94 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 77529 T94 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 77530 T94 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 77530 T94 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 77530 T94 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: solr.PingRequestHandler
   [junit4]   2> 77532 T94 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 77535 T94 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 77535 T94 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 77536 T94 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1718107 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty1/index),segFN=segments_1,generation=1}
   [junit4]   2> 77536 T94 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 77536 T94 oass.SolrIndexSearcher.<init> Opening Searcher@8887c2[collection1] main
   [junit4]   2> 77537 T95 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@8887c2[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 77537 T94 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 77538 T40 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0
   [junit4]   2> 77538 T40 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 77538 T98 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:38879/f_/y collection:collection1 shard:shard2
   [junit4]   2> 77540 T98 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 77548 T98 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 77555 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77555 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77555 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77556 T98 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 77556 T98 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C4 name=collection1 org.apache.solr.core.SolrCore@b6fd8f url=https://127.0.0.1:38879/f_/y/collection1 node=127.0.0.1:38879_f_%2Fy C4_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:38879/f_/y, core=collection1, node_name=127.0.0.1:38879_f_%2Fy}
   [junit4]   2> 77556 T98 C4 P38879 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:38879/f_/y/collection1/
   [junit4]   2> 77557 T98 C4 P38879 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 77557 T98 C4 P38879 oasc.SyncStrategy.syncToMe https://127.0.0.1:38879/f_/y/collection1/ has no replicas
   [junit4]   2> 77557 T72 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 77557 T98 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:38879/f_/y/collection1/ shard2
   [junit4]   2> 77558 T98 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 77561 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77565 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77569 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77577 T40 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 77578 T40 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 77582 T40 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 77583 T40 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:60213
   [junit4]   2> 77585 T40 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 77585 T40 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 77586 T40 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1393389323774
   [junit4]   2> 77586 T40 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1393389323774/'
   [junit4]   2> 77603 T40 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1393389323774/solr.xml
   [junit4]   2> 77648 T40 oasc.CoreContainer.<init> New CoreContainer 21007373
   [junit4]   2> 77648 T40 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1393389323774/]
   [junit4]   2> 77650 T40 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 77650 T40 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 77651 T40 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 77651 T40 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 77652 T40 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 77652 T40 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 77653 T40 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 77653 T40 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 77653 T40 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 77660 T40 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 77661 T40 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 77661 T40 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 77662 T40 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:60544/solr
   [junit4]   2> 77662 T40 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 77663 T40 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 77665 T109 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1bc0a11 name:ZooKeeperConnection Watcher:127.0.0.1:60544 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 77665 T40 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 77672 T40 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 77674 T111 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e39f21 name:ZooKeeperConnection Watcher:127.0.0.1:60544/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 77674 T40 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 77703 T71 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 77704 T93 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 77703 T80 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 77713 T40 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 77716 T98 oasc.ZkController.register We are https://127.0.0.1:38879/f_/y/collection1/ and leader is https://127.0.0.1:38879/f_/y/collection1/
   [junit4]   2> 77716 T98 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:38879/f_/y
   [junit4]   2> 77716 T98 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 77717 T98 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 77717 T98 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 77718 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77718 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77718 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77719 T98 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 77719 T72 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 77720 T72 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:38879/f_/y",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38879_f_%2Fy",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 77724 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77826 T71 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 77827 T80 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 77826 T93 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 77826 T111 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 78717 T40 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60213_f_%2Fy
   [junit4]   2> 78718 T40 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:60213_f_%2Fy
   [junit4]   2> 78720 T93 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 78720 T111 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 78721 T71 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 78720 T80 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 78728 T112 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 78728 T112 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 78729 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 78729 T112 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 78729 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 78730 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 78732 T72 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 78733 T72 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:60213/f_/y",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:60213_f_%2Fy",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 78733 T72 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 78733 T72 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 78735 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 78847 T71 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 78847 T93 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 78847 T80 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 78847 T111 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 79730 T112 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 79730 T112 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1393389323774/collection1
   [junit4]   2> 79730 T112 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 79731 T112 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 79731 T112 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 79732 T112 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 79733 T112 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1393389323774/collection1/'
   [junit4]   2> 79767 T112 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 79789 T112 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 79791 T112 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 79798 T112 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 80013 T112 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 80015 T112 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 80016 T112 oass.IndexSchema.readSchema WARN Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 80017 T112 oass.IndexSchema.readSchema WARN Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 80017 T112 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 80018 T112 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1393389323774/collection1/, dataDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty2/
   [junit4]   2> 80018 T112 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 80018 T112 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty2
   [junit4]   2> 80019 T112 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty2/index/
   [junit4]   2> 80019 T112 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 80019 T112 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty2/index
   [junit4]   2> 80020 T112 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1c8de42 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty2/index),segFN=segments_1,generation=1}
   [junit4]   2> 80020 T112 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 80021 T112 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 80021 T112 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 80022 T112 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 80022 T112 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 80022 T112 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 80022 T112 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 80023 T112 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 80023 T112 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: solr.PingRequestHandler
   [junit4]   2> 80025 T112 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 80027 T112 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 80027 T112 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 80028 T112 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1c8de42 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty2/index),segFN=segments_1,generation=1}
   [junit4]   2> 80028 T112 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 80028 T112 oass.SolrIndexSearcher.<init> Opening Searcher@15bde0d[collection1] main
   [junit4]   2> 80029 T113 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@15bde0d[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 80029 T112 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 80030 T40 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0
   [junit4]   2> 80030 T40 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 80030 T116 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:60213/f_/y collection:collection1 shard:shard1
   [junit4]   2> 80032 T116 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 80039 T116 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 80041 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80041 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80041 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80041 T116 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 80042 T116 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C5 name=collection1 org.apache.solr.core.SolrCore@3b300c url=https://127.0.0.1:60213/f_/y/collection1 node=127.0.0.1:60213_f_%2Fy C5_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:60213/f_/y, core=collection1, node_name=127.0.0.1:60213_f_%2Fy}
   [junit4]   2> 80042 T116 C5 P60213 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:60213/f_/y/collection1/
   [junit4]   2> 80043 T72 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 80043 T116 C5 P60213 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 80043 T116 C5 P60213 oasc.SyncStrategy.syncToMe https://127.0.0.1:60213/f_/y/collection1/ has no replicas
   [junit4]   2> 80043 T116 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:60213/f_/y/collection1/ shard1
   [junit4]   2> 80044 T116 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 80046 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80049 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80052 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80062 T40 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 80063 T40 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 80066 T40 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 80067 T40 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:32906
   [junit4]   2> 80069 T40 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 80069 T40 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 80070 T40 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1393389326266
   [junit4]   2> 80070 T40 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1393389326266/'
   [junit4]   2> 80081 T40 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1393389326266/solr.xml
   [junit4]   2> 80107 T40 oasc.CoreContainer.<init> New CoreContainer 19650118
   [junit4]   2> 80108 T40 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1393389326266/]
   [junit4]   2> 80109 T40 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 80109 T40 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 80109 T40 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 80110 T40 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 80110 T40 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 80110 T40 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 80111 T40 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 80111 T40 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 80111 T40 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 80115 T40 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 80115 T40 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 80116 T40 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 80116 T40 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:60544/solr
   [junit4]   2> 80116 T40 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 80117 T40 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 80119 T127 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1bfc832 name:ZooKeeperConnection Watcher:127.0.0.1:60544 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 80119 T40 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 80122 T40 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 80123 T129 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c54321 name:ZooKeeperConnection Watcher:127.0.0.1:60544/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 80123 T40 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 80127 T40 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 80158 T93 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 80158 T129 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 80158 T111 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 80158 T80 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 80158 T71 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 80200 T116 oasc.ZkController.register We are https://127.0.0.1:60213/f_/y/collection1/ and leader is https://127.0.0.1:60213/f_/y/collection1/
   [junit4]   2> 80200 T116 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:60213/f_/y
   [junit4]   2> 80200 T116 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 80201 T116 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 80201 T116 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 80202 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80202 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80202 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80203 T116 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 80204 T72 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 80205 T72 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:60213/f_/y",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:60213_f_%2Fy",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 80207 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 80309 T71 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 80309 T129 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 80309 T80 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 80309 T93 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 80309 T111 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 81130 T40 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32906_f_%2Fy
   [junit4]   2> 81131 T40 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32906_f_%2Fy
   [junit4]   2> 81134 T93 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 81135 T111 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 81135 T80 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 81136 T71 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 81136 T129 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 81143 T130 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 81143 T130 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 81144 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 81144 T130 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 81145 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 81145 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 81146 T72 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 81147 T72 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:32906/f_/y",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32906_f_%2Fy",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 81148 T72 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 81148 T72 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 81150 T71 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 81252 T80 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 81253 T111 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 81253 T93 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 81252 T129 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 81253 T71 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 82145 T130 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 82145 T130 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1393389326266/collection1
   [junit4]   2> 82145 T130 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 82146 T130 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 82146 T130 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 82148 T130 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 82148 T130 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1393389326266/collection1/'
   [junit4]   2> 82172 T130 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 82195 T130 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 82197 T130 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 82204 T130 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 82432 T130 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 82434 T130 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 82435 T130 oass.IndexSchema.readSchema WARN Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 82436 T130 oass.IndexSchema.readSchema WARN Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 82436 T130 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 82437 T130 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1393389326266/collection1/, dataDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty3/
   [junit4]   2> 82437 T130 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 82438 T130 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty3
   [junit4]   2> 82438 T130 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty3/index/
   [junit4]   2> 82438 T130 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty3/index' doesn't exist. Creating new index...
   [junit4]   2> 82439 T130 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty3/index
   [junit4]   2> 82440 T130 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@103888d lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty3/index),segFN=segments_1,generation=1}
   [junit4]   2> 82441 T130 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 82442 T130 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 82443 T130 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 82443 T130 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 82443 T130 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 82444 T130 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 82444 T130 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 82444 T130 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 82445 T130 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: solr.PingRequestHandler
   [junit4]   2> 82448 T130 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 82457 T130 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 82457 T130 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 82458 T130 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@103888d lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty3/index),segFN=segments_1,generation=1}
   [junit4]   2> 82458 T130 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 82459 T130 oass.SolrIndexSearcher.<init> Opening Searcher@189625[collection1] main
   [junit4]   2> 82459 T131 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@189625[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 82459 T130 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 82460 T40 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0
   [junit4]   2> 82461 T40 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 82460 T134 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:32906/f_/y collection:collection1 shard:shard2
   [junit4]   2> 82465 T134 oasc.ZkController.register We are https://127.0.0.1:32906/f_/y/collection1/ and leader is https://127.0.0.1:38879/f_/y/collection1/
   [junit4]   2> 82465 T134 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:32906/f_/y
   [junit4]   2> 82465 T134 oasc.ZkController.checkRecovery Core needs to recover:collection1
   [junit4]   2> 82465 T134 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
   [junit4]   2> ASYNC  NEW_CORE C6 name=collection1 org.apache.solr.core.SolrCore@1c988ac url=https://127.0.0.1:32906/f_/y/collection1 node=127.0.0.1:32906_f_%2Fy C6_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:32906/f_/y, core=collection1, node_name=127.0.0.1:32906_f_%2Fy}
   [junit4]   2> 82470 T135 C6 P32906 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 r

[...truncated too long message...]

r watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 94256 T111 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 95178 T129 C15 P32906 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
   [junit4]   2> 95181 T129 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is shutdown
   [junit4]   2> 95181 T129 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1c988ac
   [junit4]   2> 95182 T129 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=6,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=2,cumulative_deletesById=1,cumulative_deletesByQuery=2,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=5}
   [junit4]   2> 95182 T129 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 95182 T129 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 95182 T129 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 95183 T129 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 95183 T129 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 95184 T129 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty3 [CachedDir<<refCount=0;path=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty3;done=false>>]
   [junit4]   2> 95184 T129 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty3
   [junit4]   2> 95184 T129 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty3/index [CachedDir<<refCount=0;path=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty3/index;done=false>>]
   [junit4]   2> 95184 T129 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty3/index
   [junit4]   2> 95184 T129 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 95184 T129 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 95185 T129 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 95185 T129 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 95258 T148 C13 P48924 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
   [junit4]   2> 95259 T148 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is shutdown
   [junit4]   2> 95259 T148 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@788896
   [junit4]   2> 95260 T148 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=6,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=2,cumulative_deletesById=1,cumulative_deletesByQuery=2,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=5}
   [junit4]   2> 95260 T148 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 95260 T148 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 95260 T148 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 95261 T148 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 95261 T148 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 95262 T148 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty4/index [CachedDir<<refCount=0;path=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty4/index;done=false>>]
   [junit4]   2> 95262 T148 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty4/index
   [junit4]   2> 95262 T148 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty4 [CachedDir<<refCount=0;path=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty4;done=false>>]
   [junit4]   2> 95262 T148 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1393389319462/jetty4
   [junit4]   2> 95262 T148 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 95262 T148 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> NOTE: test params are: codec=Lucene46: {multiDefault=FST41, a_t=FST41, _version_=PostingsFormat(name=FSTPulsing41), range_facet_sl=PostingsFormat(name=Memory doPackFST= false), text=PostingsFormat(name=Asserting), range_facet_si=FST41, timestamp=FST41, intDefault=PostingsFormat(name=FSTPulsing41), range_facet_l=FST41, id=PostingsFormat(name=Memory doPackFST= false)}, docValues:{}, sim=DefaultSimilarity, locale=bg_BG, timezone=Zulu
   [junit4]   2> NOTE: Linux 3.8.0-36-generic i386/Oracle Corporation 1.7.0_60-ea (32-bit)/cpus=8,threads=1,free=139280800,total=190054400
   [junit4]   2> NOTE: All tests run in this JVM: [TestXMLEscaping, TermsResponseTest, NoOpResponseParserTest, ContentStreamTest, DocumentAnalysisResponseTest, SolrExceptionTest, CloudSolrServerTest]
   [junit4] Completed on J0 in 89.55s, 2 tests, 1 error <<< FAILURES!

[...truncated 100 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:471: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:451: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:202: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:490: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1278: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:911: There were test failures: 48 suites, 280 tests, 1 error

Total time: 50 minutes 8 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.7.0_60-ea-b04 -client -XX:+UseParallelGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-4.x-Linux (32bit/jdk1.6.0_45) - Build # 9493 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/9493/
Java: 32bit/jdk1.6.0_45 -client -XX:+UseParallelGC

1 tests failed.
FAILED:  org.apache.solr.client.solrj.impl.CloudSolrServerTest.testDistribSearch

Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:40212 within 30000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:40212 within 30000 ms
	at __randomizedtesting.SeedInfo.seed([DAD9548F6E84A7AB:5B3FDA9719DBC797]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:148)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:94)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:85)
	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:89)
	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:83)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.setUp(AbstractDistribZkTestBase.java:70)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.setUp(AbstractFullDistribZkTestBase.java:200)
	at org.apache.solr.client.solrj.impl.CloudSolrServerTest.setUp(CloudSolrServerTest.java:77)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
	at java.lang.reflect.Method.invoke(Method.java:597)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:771)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:662)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:40212 within 30000 ms
	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:222)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:140)
	... 47 more




Build Log:
[...truncated 10695 lines...]
   [junit4] Suite: org.apache.solr.client.solrj.impl.CloudSolrServerTest
   [junit4]   2> 5326 T42 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 5327 T42 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 5329 T42 oasc.AbstractZkTestCase.<clinit> WARN TEST_HOME() does not exist - solrj test?
   [junit4]   2> 5334 T42 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./solrtest-CloudSolrServerTest-1393399907605
   [junit4]   2> 5336 T42 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 5338 T43 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 5437 T42 oasc.ZkTestServer.run start zk server on port:40212
   [junit4]   2> 5472 T42 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 57277 T46 oazsp.FileTxnLog.commit WARN fsync-ing the write ahead log in SyncThread:0 took 51791ms which will adversely effect operation latency. See the ZooKeeper troubleshooting guide
   [junit4]   2> 57280 T42 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 57289 T44 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1446d1b455f0000, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:662)
   [junit4]   2> 
   [junit4]   2> 57290 T42 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:40212 40212
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CloudSolrServerTest -Dtests.method=testDistribSearch -Dtests.seed=DAD9548F6E84A7AB -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=nl_BE -Dtests.timezone=Asia/Ujung_Pandang -Dtests.file.encoding=UTF-8
   [junit4] ERROR   52.1s J0 | CloudSolrServerTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:40212 within 30000 ms
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([DAD9548F6E84A7AB:5B3FDA9719DBC797]:0)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:148)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:94)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:85)
   [junit4]    > 	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:89)
   [junit4]    > 	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:83)
   [junit4]    > 	at org.apache.solr.cloud.AbstractDistribZkTestBase.setUp(AbstractDistribZkTestBase.java:70)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.setUp(AbstractFullDistribZkTestBase.java:200)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServerTest.setUp(CloudSolrServerTest.java:77)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:662)
   [junit4]    > Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:40212 within 30000 ms
   [junit4]    > 	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:222)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:140)
   [junit4]    > 	... 47 more
   [junit4]   2> 57443 T42 oas.SolrTestCaseJ4.setUp ###Starting testShutdown
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./solrtest-CloudSolrServerTest-1393399959715
   [junit4]   2> 57444 T42 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 57444 T50 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 57544 T42 oasc.ZkTestServer.run start zk server on port:42154
   [junit4]   2> 57545 T42 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 57550 T56 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18b429b name:ZooKeeperConnection Watcher:127.0.0.1:42154 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 57550 T42 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 57551 T42 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 57574 T42 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 57576 T58 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e48d65 name:ZooKeeperConnection Watcher:127.0.0.1:42154/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 57577 T42 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 57585 T42 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 57592 T42 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 57598 T42 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 57602 T42 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 57608 T42 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 57614 T42 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 57622 T42 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 57622 T42 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 57628 T42 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml because it doesn't exist
   [junit4]   2> 57629 T42 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/stopwords.txt because it doesn't exist
   [junit4]   2> 57629 T42 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/protwords.txt because it doesn't exist
   [junit4]   2> 57630 T42 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/currency.xml because it doesn't exist
   [junit4]   2> 57630 T42 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/enumsConfig.xml because it doesn't exist
   [junit4]   2> 57631 T42 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/open-exchange-rates.json because it doesn't exist
   [junit4]   2> 57632 T42 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/mapping-ISOLatin1Accent.txt because it doesn't exist
   [junit4]   2> 57632 T42 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/old_synonyms.txt because it doesn't exist
   [junit4]   2> 57633 T42 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/synonyms.txt because it doesn't exist
   [junit4]   2> 57643 T42 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 57645 T60 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3d0426 name:ZooKeeperConnection Watcher:127.0.0.1:42154/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 57645 T42 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 57654 T42 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 57656 T61 oaz.ClientCnxnSocketNIO.connect ERROR Unable to open socket to ff01:0:0:0:0:0:0:114/ff01:0:0:0:0:0:0:114:33332
   [junit4]   2> 57657 T61 oaz.ClientCnxn$SendThread.run WARN Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect java.net.SocketException: Network is unreachable
   [junit4]   2> 	at sun.nio.ch.Net.connect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.connect(SocketChannelImpl.java:532)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.registerAndConnect(ClientCnxnSocketNIO.java:266)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.connect(ClientCnxnSocketNIO.java:276)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:958)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:993)
   [junit4]   2> 
   [junit4]   2> 58759 T61 oaz.ClientCnxnSocketNIO.connect ERROR Unable to open socket to ff01:0:0:0:0:0:0:114/ff01:0:0:0:0:0:0:114:33332
   [junit4]   2> 58863 T42 oas.SolrTestCaseJ4.tearDown ###Ending testShutdown
   [junit4]   2> 58865 T42 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:42154 42154
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 58977 T42 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 53657 T41 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene3x, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=nl_BE, timezone=Asia/Ujung_Pandang
   [junit4]   2> NOTE: Linux 3.8.0-36-generic i386/Sun Microsystems Inc. 1.6.0_45 (32-bit)/cpus=8,threads=1,free=22643992,total=92733440
   [junit4]   2> NOTE: All tests run in this JVM: [DocumentAnalysisResponseTest, LargeVolumeBinaryJettyTest, TestUpdateRequestCodec, CloudSolrServerTest]
   [junit4] Completed on J0 in 54.45s, 2 tests, 1 error <<< FAILURES!

[...truncated 118 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:471: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:451: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:202: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:490: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1278: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:911: There were test failures: 48 suites, 280 tests, 1 error

Total time: 56 minutes 37 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.6.0_45 -client -XX:+UseParallelGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-4.x-Linux (32bit/jdk1.8.0-fcs-b129) - Build # 9492 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/9492/
Java: 32bit/jdk1.8.0-fcs-b129 -client -XX:+UseConcMarkSweepGC

All tests passed

Build Log:
[...truncated 1128 lines...]
   [junit4] JVM J0: stdout was not empty, see: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/build/core/test/temp/junit4-J0-20140226_052533_705.sysout
   [junit4] >>> JVM J0: stdout (verbatim) ----
   [junit4] #
   [junit4] # A fatal error has been detected by the Java Runtime Environment:
   [junit4] #
   [junit4] #  SIGSEGV (0xb) at pc=0xf6fc1371, pid=7319, tid=4090493760
   [junit4] #
   [junit4] # JRE version: Java(TM) SE Runtime Environment (8.0-b129) (build 1.8.0-b129)
   [junit4] # Java VM: Java HotSpot(TM) Client VM (25.0-b69 mixed mode linux-x86 )
   [junit4] # Problematic frame:
   [junit4] # V  [libjvm.so+0x449371]  ObjArrayKlass::oop_oop_iterate_nv(oopDesc*, ParScanWithoutBarrierClosure*)+0x61
   [junit4] #
   [junit4] # Failed to write core dump. Core dumps have been disabled. To enable core dumping, try "ulimit -c unlimited" before starting Java again
   [junit4] #
   [junit4] # An error report file with more information is saved as:
   [junit4] # /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/build/core/test/J0/hs_err_pid7319.log
   [junit4] #
   [junit4] # If you would like to submit a bug report, please visit:
   [junit4] #   http://bugreport.sun.com/bugreport/crash.jsp
   [junit4] #
   [junit4] <<< JVM J0: EOF ----

[...truncated 463 lines...]
   [junit4] ERROR: JVM J0 ended with an exception, command line: /var/lib/jenkins/tools/java/32bit/jdk1.8.0-fcs-b129/jre/bin/java -client -XX:+UseConcMarkSweepGC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/heapdumps -Dtests.prefix=tests -Dtests.seed=BE9E9B4A59EF9492 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=europarl.lines.txt.gz -Dtests.luceneMatchVersion=4.8 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/tools/junit4/logging.properties -Dtests.nightly=false -Dtests.weekly=false -Dtests.slow=true -Dtests.asserts.gracious=false -Dtests.multiplier=3 -DtempDir=. -Djava.io.tmpdir=. -Djunit4.tempDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/build/core/test/temp -Dclover.db.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/build/clover/db -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Djava.security.policy=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/tools/junit4/tests.policy -Dlucene.version=4.8-SNAPSHOT -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.disableHdfs=true -Dfile.encoding=UTF-8 -classpath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/build/test-framework/classes/java:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/build/codecs/classes/java:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/test-framework/lib/junit-4.10.jar:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/test-framework/lib/randomizedtesting-runner-2.0.13.jar:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/build/core/classes/java:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/build/core/classes/test:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-launcher.jar:/var/lib/jenkins/.ant/lib/ivy-2.3.0.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jai.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-swing.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-oro.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jmf.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-xalan2.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-javamail.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-resolver.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-testutil.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-commons-logging.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-log4j.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-junit.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jsch.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-commons-net.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-bsf.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jdepend.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-netrexx.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-regexp.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-junit4.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-bcel.jar:/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-antlr.jar:/var/lib/jenkins/tools/java/32bit/jdk1.8.0-fcs-b129/lib/tools.jar:/var/lib/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.0.13.jar -ea:org.apache.lucene... -ea:org.apache.solr... com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/build/core/test/temp/junit4-J0-20140226_052533_705.events @/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/build/core/test/temp/junit4-J0-20140226_052533_705.suites
   [junit4] ERROR: JVM J0 ended with an exception: Forked process returned with error code: 134. Very likely a JVM crash.  Process output piped in logs above.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1373)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:119)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:911)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:908)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:744)

BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:471: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:451: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/build.xml:49: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1278: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:911: At least one slave process threw an exception, first: Forked process returned with error code: 134. Very likely a JVM crash.  Process output piped in logs above.

Total time: 17 minutes 0 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.8.0-fcs-b129 -client -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure