You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/11/04 14:52:28 UTC
[JENKINS] Lucene-Solr-4.x-Linux (64bit/jdk1.7.0_45) - Build # 8075
- Failure!
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/8075/
Java: 64bit/jdk1.7.0_45 -XX:+UseCompressedOops -XX:+UseG1GC
2 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
Error Message:
9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 1) Thread[id=831, name=qtp1545350713-831 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:744) 2) Thread[id=834, name=qtp1545350713-834 Acceptor1 SelectChannelConnector@127.0.0.1:53419, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241) at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:744) 3) Thread[id=838, name=qtp1545350713-838, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:744) 4) Thread[id=835, name=qtp1545350713-835, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:744) 5) Thread[id=830, name=HashSessionScavenger-19, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 6) Thread[id=837, name=qtp1545350713-837, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:744) 7) Thread[id=832, name=qtp1545350713-832 Selector1, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:744) 8) Thread[id=833, name=qtp1545350713-833 Acceptor0 SelectChannelConnector@127.0.0.1:53419, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:225) at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:744) 9) Thread[id=836, name=qtp1545350713-836, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:744)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:
1) Thread[id=831, name=qtp1545350713-831 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:744)
2) Thread[id=834, name=qtp1545350713-834 Acceptor1 SelectChannelConnector@127.0.0.1:53419, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241)
at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:744)
3) Thread[id=838, name=qtp1545350713-838, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:744)
4) Thread[id=835, name=qtp1545350713-835, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:744)
5) Thread[id=830, name=HashSessionScavenger-19, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
6) Thread[id=837, name=qtp1545350713-837, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:744)
7) Thread[id=832, name=qtp1545350713-832 Selector1, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:744)
8) Thread[id=833, name=qtp1545350713-833 Acceptor0 SelectChannelConnector@127.0.0.1:53419, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:225)
at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:744)
9) Thread[id=836, name=qtp1545350713-836, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:744)
at __randomizedtesting.SeedInfo.seed([D292C710CEE2418E]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=831, name=qtp1545350713-831 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:744) 2) Thread[id=834, name=qtp1545350713-834 Acceptor1 SelectChannelConnector@127.0.0.1:53419, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest] at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:102) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:744) 3) Thread[id=830, name=HashSessionScavenger-19, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 4) Thread[id=832, name=qtp1545350713-832 Selector1, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:744) 5) Thread[id=833, name=qtp1545350713-833 Acceptor0 SelectChannelConnector@127.0.0.1:53419, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest] at org.eclipse.jetty.server.nio.SelectChannelConnector.getConnection(SelectChannelConnector.java:160) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:934) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:744)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=831, name=qtp1545350713-831 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:744)
2) Thread[id=834, name=qtp1545350713-834 Acceptor1 SelectChannelConnector@127.0.0.1:53419, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:102)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:744)
3) Thread[id=830, name=HashSessionScavenger-19, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
4) Thread[id=832, name=qtp1545350713-832 Selector1, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:744)
5) Thread[id=833, name=qtp1545350713-833 Acceptor0 SelectChannelConnector@127.0.0.1:53419, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at org.eclipse.jetty.server.nio.SelectChannelConnector.getConnection(SelectChannelConnector.java:160)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:934)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:744)
at __randomizedtesting.SeedInfo.seed([D292C710CEE2418E]:0)
Build Log:
[...truncated 10019 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
[junit4] 2> 203466 T582 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_dnc/gy
[junit4] 2> 203475 T582 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4] 2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solrtest-ChaosMonkeyNothingIsSafeTest-1383572420359
[junit4] 2> 203476 T582 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 203477 T583 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 203577 T582 oasc.ZkTestServer.run start zk server on port:37014
[junit4] 2> 203579 T582 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 203616 T589 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6cc85eb9 name:ZooKeeperConnection Watcher:127.0.0.1:37014 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 203616 T582 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 203617 T582 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 203625 T582 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 203627 T591 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@71d6d6f5 name:ZooKeeperConnection Watcher:127.0.0.1:37014/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 203628 T582 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 203629 T582 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4] 2> 203635 T582 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4] 2> 203640 T582 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4] 2> 203644 T582 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4] 2> 203649 T582 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 203651 T582 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4] 2> 203656 T582 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 203657 T582 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4] 2> 203660 T582 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 203660 T582 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 203662 T582 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 203663 T582 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4] 2> 203665 T582 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 203666 T582 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4] 2> 203668 T582 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 203669 T582 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4] 2> 203671 T582 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 203671 T582 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4] 2> 203675 T582 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 203675 T582 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 203678 T582 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 203678 T582 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4] 2> 203680 T582 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 203681 T582 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4] 2> 203747 T582 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 203750 T582 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:56166
[junit4] 2> 203750 T582 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 203751 T582 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 203751 T582 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383572420566
[junit4] 2> 203751 T582 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383572420566/'
[junit4] 2> 203761 T582 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383572420566/solr.xml
[junit4] 2> 203791 T582 oasc.CoreContainer.<init> New CoreContainer 1220558861
[junit4] 2> 203791 T582 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383572420566/]
[junit4] 2> 203792 T582 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
[junit4] 2> 203792 T582 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 203793 T582 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 203793 T582 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 203793 T582 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 203793 T582 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 203793 T582 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 203794 T582 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 203794 T582 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 203795 T582 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
[junit4] 2> 203795 T582 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 203796 T582 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37014/solr
[junit4] 2> 203796 T582 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 203796 T582 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 203798 T602 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@36429c28 name:ZooKeeperConnection Watcher:127.0.0.1:37014 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 203798 T582 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 203802 T582 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 203803 T604 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@30463be7 name:ZooKeeperConnection Watcher:127.0.0.1:37014/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 203804 T582 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 203806 T582 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 203810 T582 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 203813 T582 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 203815 T582 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56166__dnc%2Fgy
[junit4] 2> 203816 T582 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:56166__dnc%2Fgy
[junit4] 2> 203822 T582 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 203832 T582 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 203837 T582 oasc.Overseer.start Overseer (id=90673802140844035-127.0.0.1:56166__dnc%2Fgy-n_0000000000) starting
[junit4] 2> 203843 T582 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 203849 T606 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 203850 T582 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 203855 T582 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 203858 T582 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 203864 T605 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 203870 T607 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 203871 T607 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 203873 T607 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 205369 T605 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 205370 T605 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:56166/_dnc/gy",
[junit4] 2> "core":"collection1",
[junit4] 2> "roles":null,
[junit4] 2> "node_name":"127.0.0.1:56166__dnc%2Fgy",
[junit4] 2> "shard":null,
[junit4] 2> "shard_range":null,
[junit4] 2> "shard_state":"active",
[junit4] 2> "shard_parent":null,
[junit4] 2> "collection":"control_collection",
[junit4] 2> "numShards":"1",
[junit4] 2> "core_node_name":null}
[junit4] 2> 205371 T605 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
[junit4] 2> 205371 T605 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 205378 T604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 205873 T607 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
[junit4] 2> 205874 T607 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383572420566/collection1
[junit4] 2> 205874 T607 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4] 2> 205875 T607 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4] 2> 205875 T607 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4] 2> 205876 T607 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383572420566/collection1/'
[junit4] 2> 205876 T607 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383572420566/collection1/lib/classes/' to classloader
[junit4] 2> 205877 T607 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383572420566/collection1/lib/README' to classloader
[junit4] 2> 205907 T607 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_46
[junit4] 2> 205939 T607 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4] 2> 205941 T607 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4] 2> 205949 T607 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4] 2> 206268 T607 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
[junit4] 2> 206269 T607 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
[junit4] 2> 206270 T607 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4] 2> 206277 T607 oass.IndexSchema.readSchema default search field in schema is text
[junit4] 2> 206279 T607 oass.IndexSchema.readSchema unique key field: id
[junit4] 2> 206299 T607 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 206303 T607 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 206317 T607 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 206319 T607 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 206319 T607 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 206319 T607 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 206320 T607 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 206320 T607 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 206320 T607 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4] 2> 206326 T607 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383572420566/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/control/data/
[junit4] 2> 206326 T607 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@71c66aab
[junit4] 2> 206327 T607 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/control/data
[junit4] 2> 206327 T607 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/control/data/index/
[junit4] 2> 206327 T607 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/control/data/index' doesn't exist. Creating new index...
[junit4] 2> 206328 T607 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/control/data/index
[junit4] 2> 206328 T607 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=13, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 206330 T607 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4] 2> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/control/data/index,segFN=segments_1,generation=1}
[junit4] 2> 206330 T607 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 206333 T607 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
[junit4] 2> 206334 T607 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
[junit4] 2> 206334 T607 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4] 2> 206334 T607 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
[junit4] 2> 206334 T607 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4] 2> 206335 T607 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
[junit4] 2> 206335 T607 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 206335 T607 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 206335 T607 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
[junit4] 2> 206336 T607 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4] 2> 206336 T607 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4] 2> 206336 T607 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4] 2> 206337 T607 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4] 2> 206337 T607 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4] 2> 206337 T607 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4] 2> 206339 T607 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4] 2> 206341 T607 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4] 2> 206342 T607 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4] 2> 206342 T607 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=15, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 206343 T607 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4] 2> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/control/data/index,segFN=segments_1,generation=1}
[junit4] 2> 206343 T607 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 206344 T607 oass.SolrIndexSearcher.<init> Opening Searcher@19500962 main
[junit4] 2> 206346 T608 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@19500962 main{StandardDirectoryReader(segments_1:1:nrt)}
[junit4] 2> 206347 T607 oasc.CoreContainer.registerCore registering core: collection1
[junit4] 2> 206347 T607 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:56166/_dnc/gy collection:control_collection shard:shard1
[junit4] 2> 206348 T607 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4] 2> 206356 T607 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
[junit4] 2> 206358 T607 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4] 2> 206358 T607 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4] 2> 206358 T607 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:56166/_dnc/gy/collection1/
[junit4] 2> 206359 T607 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4] 2> 206359 T607 oasc.SyncStrategy.syncToMe http://127.0.0.1:56166/_dnc/gy/collection1/ has no replicas
[junit4] 2> 206359 T607 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:56166/_dnc/gy/collection1/ shard1
[junit4] 2> 206359 T607 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4] 2> 206884 T605 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 206899 T604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 206917 T607 oasc.ZkController.register We are http://127.0.0.1:56166/_dnc/gy/collection1/ and leader is http://127.0.0.1:56166/_dnc/gy/collection1/
[junit4] 2> 206918 T607 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:56166/_dnc/gy
[junit4] 2> 206918 T607 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4] 2> 206918 T607 oasc.ZkController.publish publishing core=collection1 state=active
[junit4] 2> 206919 T607 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 206965 T607 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 206967 T582 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
[junit4] 2> 206968 T582 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4] 2> 206971 T582 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 206974 T611 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29642676 name:ZooKeeperConnection Watcher:127.0.0.1:37014/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 206975 T582 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 206977 T582 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 206983 T582 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 207057 T582 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 207060 T582 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:44043
[junit4] 2> 207061 T582 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 207062 T582 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 207062 T582 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383572423867
[junit4] 2> 207063 T582 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383572423867/'
[junit4] 2> 207079 T582 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383572423867/solr.xml
[junit4] 2> 207109 T582 oasc.CoreContainer.<init> New CoreContainer 1655795691
[junit4] 2> 207109 T582 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383572423867/]
[junit4] 2> 207110 T582 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
[junit4] 2> 207110 T582 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 207111 T582 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 207111 T582 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 207111 T582 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 207111 T582 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 207112 T582 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 207112 T582 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 207112 T582 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 207113 T582 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
[junit4] 2> 207113 T582 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 207113 T582 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37014/solr
[junit4] 2> 207114 T582 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 207114 T582 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 207116 T622 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b297517 name:ZooKeeperConnection Watcher:127.0.0.1:37014 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 207117 T582 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 207120 T582 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 207123 T624 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78639899 name:ZooKeeperConnection Watcher:127.0.0.1:37014/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 207123 T582 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 207130 T582 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 208136 T582 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44043__dnc%2Fgy
[junit4] 2> 208138 T582 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:44043__dnc%2Fgy
[junit4] 2> 208144 T604 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 208144 T624 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 208144 T611 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 208155 T625 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 208156 T625 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 208158 T625 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 208407 T605 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 208408 T605 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "base_url":"http://127.0.0.1:56166/_dnc/gy",
[junit4] 2> "core":"collection1",
[junit4] 2> "roles":null,
[junit4] 2> "node_name":"127.0.0.1:56166__dnc%2Fgy",
[junit4] 2> "shard":"shard1",
[junit4] 2> "shard_range":null,
[junit4] 2> "shard_state":"active",
[junit4] 2> "shard_parent":null,
[junit4] 2> "collection":"control_collection",
[junit4] 2> "numShards":"1",
[junit4] 2> "core_node_name":"core_node1"}
[junit4] 2> 208412 T605 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:44043/_dnc/gy",
[junit4] 2> "core":"collection1",
[junit4] 2> "roles":null,
[junit4] 2> "node_name":"127.0.0.1:44043__dnc%2Fgy",
[junit4] 2> "shard":null,
[junit4] 2> "shard_range":null,
[junit4] 2> "shard_state":"active",
[junit4] 2> "shard_parent":null,
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"2",
[junit4] 2> "core_node_name":null}
[junit4] 2> 208412 T605 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
[junit4] 2> 208412 T605 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 208418 T611 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 208418 T604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 208418 T624 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 209158 T625 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
[junit4] 2> 209159 T625 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383572423867/collection1
[junit4] 2> 209159 T625 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4] 2> 209159 T625 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4] 2> 209160 T625 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4] 2> 209161 T625 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383572423867/collection1/'
[junit4] 2> 209161 T625 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383572423867/collection1/lib/classes/' to classloader
[junit4] 2> 209162 T625 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383572423867/collection1/lib/README' to classloader
[junit4] 2> 209199 T625 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_46
[junit4] 2> 209245 T625 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4] 2> 209247 T625 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4] 2> 209258 T625 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4] 2> 209595 T625 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
[junit4] 2> 209595 T625 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
[junit4] 2> 209595 T625 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4] 2> 209602 T625 oass.IndexSchema.readSchema default search field in schema is text
[junit4] 2> 209604 T625 oass.IndexSchema.readSchema unique key field: id
[junit4] 2> 209619 T625 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 209622 T625 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 209626 T625 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 209627 T625 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 209627 T625 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 209627 T625 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 209628 T625 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 209628 T625 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 209629 T625 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4] 2> 209629 T625 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383572423867/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/jetty1/
[junit4] 2> 209629 T625 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@71c66aab
[junit4] 2> 209630 T625 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/jetty1
[junit4] 2> 209630 T625 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/jetty1/index/
[junit4] 2> 209630 T625 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/jetty1/index' doesn't exist. Creating new index...
[junit4] 2> 209630 T625 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/jetty1/index
[junit4] 2> 209631 T625 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=13, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 209647 T625 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4] 2> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/jetty1/index,segFN=segments_1,generation=1}
[junit4] 2> 209647 T625 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 209653 T625 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
[junit4] 2> 209653 T625 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
[junit4] 2> 209653 T625 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4] 2> 209653 T625 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
[junit4] 2> 209654 T625 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4] 2> 209654 T625 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
[junit4] 2> 209654 T625 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 209655 T625 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 209655 T625 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
[junit4] 2> 209656 T625 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4] 2> 209656 T625 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4] 2> 209656 T625 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4] 2> 209657 T625 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4] 2> 209657 T625 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4] 2> 209658 T625 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4] 2> 209660 T625 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4] 2> 209663 T625 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4] 2> 209663 T625 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4] 2> 209664 T625 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=15, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 209665 T625 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4] 2> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/jetty1/index,segFN=segments_1,generation=1}
[junit4] 2> 209665 T625 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 209665 T625 oass.SolrIndexSearcher.<init> Opening Searcher@337844f5 main
[junit4] 2> 209669 T626 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@337844f5 main{StandardDirectoryReader(segments_1:1:nrt)}
[junit4] 2> 209671 T625 oasc.CoreContainer.registerCore registering core: collection1
[junit4] 2> 209672 T625 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:44043/_dnc/gy collection:collection1 shard:shard1
[junit4] 2> 209673 T625 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 209680 T625 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
[junit4] 2> 209682 T625 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4] 2> 209683 T625 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4] 2> 209683 T625 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:44043/_dnc/gy/collection1/
[junit4] 2> 209683 T625 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4] 2> 209683 T625 oasc.SyncStrategy.syncToMe http://127.0.0.1:44043/_dnc/gy/collection1/ has no replicas
[junit4] 2> 209683 T625 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:44043/_dnc/gy/collection1/ shard1
[junit4] 2> 209684 T625 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 209924 T605 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 209932 T611 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 209932 T604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 209932 T624 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 209941 T625 oasc.ZkController.register We are http://127.0.0.1:44043/_dnc/gy/collection1/ and leader is http://127.0.0.1:44043/_dnc/gy/collection1/
[junit4] 2> 209941 T625 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:44043/_dnc/gy
[junit4] 2> 209941 T625 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4] 2> 209942 T625 oasc.ZkController.publish publishing core=collection1 state=active
[junit4] 2> 209942 T625 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 209943 T625 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 209945 T582 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
[junit4] 2> 209945 T582 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4] 2> 210013 T582 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 210015 T582 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53419
[junit4] 2> 210016 T582 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 210016 T582 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 210016 T582 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383572426829
[junit4] 2> 210016 T582 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383572426829/'
[junit4] 2> 210026 T582 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383572426829/solr.xml
[junit4] 2> 210058 T582 oasc.CoreContainer.<init> New CoreContainer 2009297016
[junit4] 2> 210058 T582 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383572426829/]
[junit4] 2> 210059 T582 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
[junit4] 2> 210059 T582 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 210059 T582 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 210060 T582 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 210060 T582 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 210060 T582 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 210060 T582 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 210061 T582 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 210061 T582 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 210062 T582 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
[junit4] 2> 210062 T582 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 210063 T582 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37014/solr
[junit4] 2> 210063 T582 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 210064 T582 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 210065 T638 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14816537 name:ZooKeeperConnection Watcher:127.0.0.1:37014 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 210065 T582 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 210068 T582 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 210069 T640 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@61fa1e77 name:ZooKeeperConnection Watcher:127.0.0.1:37014/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 210070 T582 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 210074 T582 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 211078 T582 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53419__dnc%2Fgy
[junit4] 2> 211081 T582 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53419__dnc%2Fgy
[junit4] 2> 211090 T624 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 211090 T611 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 211090 T640 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 211090 T604 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 211101 T641 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 211102 T641 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 211104 T641 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 211440 T605 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 211441 T605 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "base_url":"http://127.0.0.1:44043/_dnc/gy",
[junit4] 2> "core":"collection1",
[junit4] 2> "roles":null,
[junit4] 2> "node_name":"127.0.0.1:44043__dnc%2Fgy",
[junit4] 2> "shard":"shard1",
[junit4] 2> "shard_range":null,
[junit4] 2> "shard_state":"active",
[junit4] 2> "shard_parent":null,
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"2",
[junit4] 2> "core_node_name":"core_node1"}
[junit4] 2> 211449 T605 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:53419/_dnc/gy",
[junit4] 2> "core":"collection1",
[junit4] 2> "roles":null,
[junit4] 2> "node_name":"127.0.0.1:53419__dnc%2Fgy",
[junit4] 2> "shard":null,
[junit4] 2> "shard_range":null,
[junit4] 2> "shard_state":"active",
[junit4] 2> "shard_parent":null,
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"2",
[junit4] 2> "core_node_name":null}
[junit4] 2> 211449 T605 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4] 2> 211450 T605 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
[junit4] 2> 211457 T611 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 211458 T624 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 211458 T640 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 211458 T604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 212104 T641 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
[junit4] 2> 212105 T641 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383572426829/collection1
[junit4] 2> 212105 T641 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4] 2> 212106 T641 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4] 2> 212106 T641 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4] 2> 212107 T641 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383572426829/collection1/'
[junit4] 2> 212108 T641 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383572426829/collection1/lib/classes/' to classloader
[junit4] 2> 212108 T641 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383572426829/collection1/lib/README' to classloader
[junit4] 2> 212138 T641 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_46
[junit4] 2> 212168 T641 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4] 2> 212169 T641 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4] 2> 212176 T641 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4] 2> 212528 T641 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
[junit4] 2> 212528 T641 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
[junit4] 2> 212529 T641 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4] 2> 212534 T641 oass.IndexSchema.readSchema default search field in schema is text
[junit4] 2> 212537 T641 oass.IndexSchema.readSchema unique key field: id
[junit4] 2> 212553 T641 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 212557 T641 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 212560 T641 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 212561 T641 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 212561 T641 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 212562 T641 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 212563 T641 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 212563 T641 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 212563 T641 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4] 2> 212563 T641 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383572426829/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/jetty2/
[junit4] 2> 212564 T641 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@71c66aab
[junit4] 2> 212564 T641 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/jetty2
[junit4] 2> 212565 T641 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/jetty2/index/
[junit4] 2> 212565 T641 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/jetty2/index' doesn't exist. Creating new index...
[junit4] 2> 212565 T641 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/jetty2/index
[junit4] 2> 212566 T641 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=13, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 212569 T641 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4] 2> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/jetty2/index,segFN=segments_1,generation=1}
[junit4] 2> 212569 T641 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 212573 T641 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
[junit4] 2> 212573 T641 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
[junit4] 2> 212573 T641 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4] 2> 212573 T641 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
[junit4] 2> 212574 T641 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4] 2> 212574 T641 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
[junit4] 2> 212574 T641 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 212574 T641 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 212575 T641 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
[junit4] 2> 212575 T641 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4] 2> 212576 T641 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4] 2> 212576 T641 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4] 2> 212576 T641 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4] 2> 212577 T641 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4] 2> 212577 T641 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4] 2> 212579 T641 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4] 2> 212581 T641 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4] 2> 212581 T641 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4] 2> 212582 T641 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=15, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 212583 T641 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4] 2> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383572420359/jetty2/index,segFN=segments_1,generation=1}
[junit4] 2> 212583 T641 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 212583 T641 oass.SolrIndexSearcher.<init> Opening Searcher@3100726d main
[junit4] 2> 212587 T642 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3100726d main{StandardDirectoryReader(segments_1:1:nrt)}
[junit4] 2> 212589 T641 oasc.CoreContainer.registerCore registering core: collection1
[junit4] 2> 212589 T641 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:53419/_dnc/gy collection:collection1 shard:shard2
[junit4] 2> 212590 T641 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4] 2> 212596 T641 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
[junit4] 2> 212598 T641 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4] 2> 212598 T641 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4] 2> 212598 T641 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:53419/_dnc/gy/collection1/
[junit4] 2> 212599 T641 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4] 2> 212599 T641 oasc.SyncStrategy.syncToMe http://127.0.0.1:53419/_dnc/gy/collection1/ has no replicas
[junit4] 2> 212599 T641 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:53419/_dnc/gy/collection1/ shard2
[junit4] 2> 212599 T641 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4] 2> 212967 T605 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 212976 T611 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 212976 T624 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 212976 T640 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 212976 T604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 213005 T641 oasc.ZkController.register We are http://127.0.0.1:53419/_dnc/gy/collection1/ and leader is http://127.0.0.1:53419/_dnc/gy/collection1/
[junit4] 2> 213006 T641 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:53419/_dnc/gy
[junit4] 2> 213006 T641 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4] 2> 213006 T641 oasc.ZkController.publish publishing core=collection1 state=active
[junit4] 2> 213006 T641 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 213008 T641 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 213009 T582 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
[junit4] 2> 213009 T582 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4] 2> 213078 T582 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 213081 T582 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:36034
[junit4] 2> 213082 T582 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 213082 T582 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 213082 T582 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty3-1383572429893
[junit4] 2> 213083 T582 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty3-1383572429893/'
[junit4] 2> 213092 T582 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty3-1383572429893/solr.xml
[junit4] 2> 213123 T582 oasc.CoreContainer.<init> New CoreContainer 1665441873
[junit4] 2> 213124 T582 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty3-1383572429893/]
[junit4] 2> 213124 T582 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
[junit4] 2> 213125 T582 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 213125 T582 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 213125 T582 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 213125 T582 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 213126 T582 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 213126 T582 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 213126 T582 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 213126 T582 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 213127 T582 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
[junit4] 2> 213128 T582 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 213128 T582 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37014/solr
[junit4] 2> 213128 T582 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 213129 T582 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 213131 T654 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5cb3b10 name:ZooKeeperConnection Watcher:127.0.0.1:37014 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 213131 T582 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 213134 T582 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 213135 T656 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@eebf029 name:ZooKeeperConnection Watcher:127.0.0.1:37014/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 213135 T582 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 213138 T582 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 214142 T582 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36034__dnc%2Fgy
[junit4] 2> 214144 T582 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:36034__dnc%2Fgy
[junit4] 2> 214149 T611 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 214150 T604 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 214150 T656 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 214150 T640 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 214150 T624 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 214163 T657 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 214163 T657 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 214168 T657 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 214484 T605 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 214486 T605 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "base_url":"http://127.0.0.1:53419/_dnc/gy",
[junit4] 2> "core":"collection1",
[junit4] 2> "roles":null,
[junit4] 2> "node_name":"127.0.0.1:53419__dnc%2Fgy",
[junit4] 2> "shard":"shard2",
[junit4] 2> "shard_range":null,
[junit4] 2> "shard_state":"active",
[junit4] 2> "shard_paren
[...truncated too long message...]
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
[junit4] > at java.lang.Thread.run(Thread.java:744)
[junit4] > 5) Thread[id=830, name=HashSessionScavenger-19, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.util.TimerThread.mainLoop(Timer.java:552)
[junit4] > at java.util.TimerThread.run(Timer.java:505)
[junit4] > 6) Thread[id=837, name=qtp1545350713-837, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
[junit4] > at java.lang.Thread.run(Thread.java:744)
[junit4] > 7) Thread[id=832, name=qtp1545350713-832 Selector1, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
[junit4] > at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
[junit4] > at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:744)
[junit4] > 8) Thread[id=833, name=qtp1545350713-833 Acceptor0 SelectChannelConnector@127.0.0.1:53419, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:225)
[junit4] > at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
[junit4] > at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:744)
[junit4] > 9) Thread[id=836, name=qtp1545350713-836, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
[junit4] > at java.lang.Thread.run(Thread.java:744)
[junit4] > at __randomizedtesting.SeedInfo.seed([D292C710CEE2418E]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=831, name=qtp1545350713-831 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
[junit4] > at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
[junit4] > at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:744)
[junit4] > 2) Thread[id=834, name=qtp1545350713-834 Acceptor1 SelectChannelConnector@127.0.0.1:53419, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:102)
[junit4] > at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:744)
[junit4] > 3) Thread[id=830, name=HashSessionScavenger-19, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.util.TimerThread.mainLoop(Timer.java:552)
[junit4] > at java.util.TimerThread.run(Timer.java:505)
[junit4] > 4) Thread[id=832, name=qtp1545350713-832 Selector1, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
[junit4] > at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
[junit4] > at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:744)
[junit4] > 5) Thread[id=833, name=qtp1545350713-833 Acceptor0 SelectChannelConnector@127.0.0.1:53419, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at org.eclipse.jetty.server.nio.SelectChannelConnector.getConnection(SelectChannelConnector.java:160)
[junit4] > at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:934)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:744)
[junit4] > at __randomizedtesting.SeedInfo.seed([D292C710CEE2418E]:0)
[junit4] Completed on J1 in 164.39s, 1 test, 2 errors <<< FAILURES!
[...truncated 870 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:428: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:408: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:483: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1263: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:906: There were test failures: 333 suites, 1469 tests, 2 suite-level errors, 841 ignored (4 assumptions)
Total time: 36 minutes 41 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0_45 -XX:+UseCompressedOops -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure
[JENKINS] Lucene-Solr-4.x-Linux (64bit/jdk1.6.0_45) - Build # 8076
- Still Failing!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/8076/
Java: 64bit/jdk1.6.0_45 -XX:+UseCompressedOops -XX:+UseSerialGC
2 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
Error Message:
9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 1) Thread[id=2027, name=qtp1642456440-2027 Selector1, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:662) 2) Thread[id=2026, name=qtp1642456440-2026 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:662) 3) Thread[id=2028, name=qtp1642456440-2028 Acceptor0 SelectChannelConnector@127.0.0.1:48168, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:150) at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:662) 4) Thread[id=2025, name=HashSessionScavenger-65, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:509) at java.util.TimerThread.run(Timer.java:462) 5) Thread[id=2031, name=qtp1642456440-2031, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:662) 6) Thread[id=2032, name=qtp1642456440-2032, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:662) 7) Thread[id=2029, name=qtp1642456440-2029 Acceptor1 SelectChannelConnector@127.0.0.1:48168, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:134) at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:662) 8) Thread[id=2030, name=qtp1642456440-2030, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:662) 9) Thread[id=2033, name=qtp1642456440-2033, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:662)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:
1) Thread[id=2027, name=qtp1642456440-2027 Selector1, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:662)
2) Thread[id=2026, name=qtp1642456440-2026 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:662)
3) Thread[id=2028, name=qtp1642456440-2028 Acceptor0 SelectChannelConnector@127.0.0.1:48168, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:150)
at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:662)
4) Thread[id=2025, name=HashSessionScavenger-65, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:509)
at java.util.TimerThread.run(Timer.java:462)
5) Thread[id=2031, name=qtp1642456440-2031, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:662)
6) Thread[id=2032, name=qtp1642456440-2032, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:662)
7) Thread[id=2029, name=qtp1642456440-2029 Acceptor1 SelectChannelConnector@127.0.0.1:48168, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:134)
at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:662)
8) Thread[id=2030, name=qtp1642456440-2030, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:662)
9) Thread[id=2033, name=qtp1642456440-2033, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:662)
at __randomizedtesting.SeedInfo.seed([C13BD390334B6218]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=2027, name=qtp1642456440-2027 Selector1, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:662) 2) Thread[id=2026, name=qtp1642456440-2026 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:662) 3) Thread[id=2028, name=qtp1642456440-2028 Acceptor0 SelectChannelConnector@127.0.0.1:48168, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest] at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:956) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:662) 4) Thread[id=2025, name=HashSessionScavenger-65, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:509) at java.util.TimerThread.run(Timer.java:462) 5) Thread[id=2029, name=qtp1642456440-2029 Acceptor1 SelectChannelConnector@127.0.0.1:48168, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest] at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:102) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:662)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=2027, name=qtp1642456440-2027 Selector1, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:662)
2) Thread[id=2026, name=qtp1642456440-2026 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:662)
3) Thread[id=2028, name=qtp1642456440-2028 Acceptor0 SelectChannelConnector@127.0.0.1:48168, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:956)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:662)
4) Thread[id=2025, name=HashSessionScavenger-65, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:509)
at java.util.TimerThread.run(Timer.java:462)
5) Thread[id=2029, name=qtp1642456440-2029 Acceptor1 SelectChannelConnector@127.0.0.1:48168, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:102)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:662)
at __randomizedtesting.SeedInfo.seed([C13BD390334B6218]:0)
Build Log:
[...truncated 9600 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
[junit4] 2> 533553 T1815 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
[junit4] 2> 533567 T1815 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4] 2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solrtest-ChaosMonkeyNothingIsSafeTest-1383577960424
[junit4] 2> 533567 T1815 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 533568 T1816 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 533668 T1815 oasc.ZkTestServer.run start zk server on port:60232
[junit4] 2> 533670 T1815 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 533765 T1822 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@27fde451 name:ZooKeeperConnection Watcher:127.0.0.1:60232 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 533766 T1815 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 533766 T1815 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 533776 T1815 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 533778 T1824 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fc40e40 name:ZooKeeperConnection Watcher:127.0.0.1:60232/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 533778 T1815 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 533779 T1815 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4] 2> 533783 T1815 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4] 2> 533786 T1815 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4] 2> 533789 T1815 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4] 2> 533794 T1815 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 533794 T1815 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4] 2> 533800 T1815 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 533801 T1815 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4] 2> 533805 T1815 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 533806 T1815 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 533810 T1815 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 533810 T1815 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4] 2> 533814 T1815 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 533815 T1815 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4] 2> 533819 T1815 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 533819 T1815 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4] 2> 533823 T1815 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 533823 T1815 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4] 2> 533827 T1815 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 533828 T1815 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 533832 T1815 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 533832 T1815 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4] 2> 533836 T1815 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 533836 T1815 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4] 2> 533917 T1815 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 533920 T1815 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:55584
[junit4] 2> 533921 T1815 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 533921 T1815 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 533921 T1815 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383577960698
[junit4] 2> 533922 T1815 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383577960698/'
[junit4] 2> 533931 T1815 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383577960698/solr.xml
[junit4] 2> 533973 T1815 oasc.CoreContainer.<init> New CoreContainer 802995013
[junit4] 2> 533974 T1815 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383577960698/]
[junit4] 2> 533975 T1815 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
[junit4] 2> 533975 T1815 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 533976 T1815 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 533976 T1815 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 533976 T1815 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 533977 T1815 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 533977 T1815 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 533977 T1815 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 533977 T1815 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 533979 T1815 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
[junit4] 2> 533979 T1815 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 533979 T1815 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:60232/solr
[junit4] 2> 533980 T1815 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 533981 T1815 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 533983 T1835 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@448ac78f name:ZooKeeperConnection Watcher:127.0.0.1:60232 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 533984 T1815 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 533987 T1815 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 533988 T1837 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38769edd name:ZooKeeperConnection Watcher:127.0.0.1:60232/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 533988 T1815 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 533990 T1815 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 533994 T1815 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 533999 T1815 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 534001 T1815 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55584_
[junit4] 2> 534002 T1815 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55584_
[junit4] 2> 534006 T1815 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 534013 T1815 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 534016 T1815 oasc.Overseer.start Overseer (id=90674165214478339-127.0.0.1:55584_-n_0000000000) starting
[junit4] 2> 534020 T1815 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 534024 T1839 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 534025 T1815 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 534029 T1815 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 534032 T1815 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 534037 T1838 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 534040 T1840 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 534040 T1840 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 534042 T1840 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 535543 T1838 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 535544 T1838 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:55584",
[junit4] 2> "core":"collection1",
[junit4] 2> "roles":null,
[junit4] 2> "node_name":"127.0.0.1:55584_",
[junit4] 2> "shard":null,
[junit4] 2> "shard_range":null,
[junit4] 2> "shard_state":"active",
[junit4] 2> "shard_parent":null,
[junit4] 2> "collection":"control_collection",
[junit4] 2> "numShards":"1",
[junit4] 2> "core_node_name":null}
[junit4] 2> 535544 T1838 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
[junit4] 2> 535544 T1838 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 535550 T1837 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 536042 T1840 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
[junit4] 2> 536042 T1840 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383577960698/collection1
[junit4] 2> 536043 T1840 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4] 2> 536044 T1840 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4] 2> 536044 T1840 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4] 2> 536046 T1840 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383577960698/collection1/'
[junit4] 2> 536046 T1840 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383577960698/collection1/lib/classes/' to classloader
[junit4] 2> 536047 T1840 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383577960698/collection1/lib/README' to classloader
[junit4] 2> 536086 T1840 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_46
[junit4] 2> 536118 T1840 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4] 2> 536120 T1840 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4] 2> 536126 T1840 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4] 2> 536491 T1840 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
[junit4] 2> 536491 T1840 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
[junit4] 2> 536492 T1840 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4] 2> 536498 T1840 oass.IndexSchema.readSchema default search field in schema is text
[junit4] 2> 536500 T1840 oass.IndexSchema.readSchema unique key field: id
[junit4] 2> 536517 T1840 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 536521 T1840 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 536524 T1840 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 536526 T1840 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 536526 T1840 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 536526 T1840 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 536528 T1840 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 536528 T1840 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 536528 T1840 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4] 2> 536529 T1840 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-controljetty-1383577960698/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/control/data/
[junit4] 2> 536529 T1840 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cea6f21
[junit4] 2> 536529 T1840 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/control/data
[junit4] 2> 536530 T1840 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/control/data/index/
[junit4] 2> 536530 T1840 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/control/data/index' doesn't exist. Creating new index...
[junit4] 2> 536530 T1840 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/control/data/index
[junit4] 2> 536530 T1840 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@274cf4b1
[junit4] 2> 536613 T1840 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4] 2> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/control/data/index,segFN=segments_1,generation=1}
[junit4] 2> 536614 T1840 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 536616 T1840 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
[junit4] 2> 536616 T1840 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
[junit4] 2> 536617 T1840 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4] 2> 536617 T1840 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
[junit4] 2> 536617 T1840 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4] 2> 536617 T1840 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
[junit4] 2> 536618 T1840 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 536618 T1840 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 536618 T1840 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
[junit4] 2> 536618 T1840 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4] 2> 536619 T1840 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4] 2> 536619 T1840 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4] 2> 536619 T1840 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4] 2> 536619 T1840 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4] 2> 536620 T1840 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4] 2> 536621 T1840 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4] 2> 536623 T1840 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4] 2> 536624 T1840 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4] 2> 536624 T1840 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=19, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 536625 T1840 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4] 2> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/control/data/index,segFN=segments_1,generation=1}
[junit4] 2> 536625 T1840 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 536626 T1840 oass.SolrIndexSearcher.<init> Opening Searcher@363f910a main
[junit4] 2> 536627 T1841 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@363f910a main{StandardDirectoryReader(segments_1:1:nrt)}
[junit4] 2> 536628 T1840 oasc.CoreContainer.registerCore registering core: collection1
[junit4] 2> 536628 T1840 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:55584 collection:control_collection shard:shard1
[junit4] 2> 536630 T1840 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4] 2> 536639 T1840 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
[junit4] 2> 536642 T1840 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4] 2> 536642 T1840 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4] 2> 536642 T1840 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:55584/collection1/
[junit4] 2> 536642 T1840 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4] 2> 536642 T1840 oasc.SyncStrategy.syncToMe http://127.0.0.1:55584/collection1/ has no replicas
[junit4] 2> 536643 T1840 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:55584/collection1/ shard1
[junit4] 2> 536643 T1840 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4] 2> 537063 T1838 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 537074 T1837 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 537102 T1840 oasc.ZkController.register We are http://127.0.0.1:55584/collection1/ and leader is http://127.0.0.1:55584/collection1/
[junit4] 2> 537102 T1840 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:55584
[junit4] 2> 537102 T1840 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4] 2> 537102 T1840 oasc.ZkController.publish publishing core=collection1 state=active
[junit4] 2> 537103 T1840 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 537105 T1840 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 537107 T1815 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
[junit4] 2> 537107 T1815 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4] 2> 537109 T1815 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 537110 T1844 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@127f641e name:ZooKeeperConnection Watcher:127.0.0.1:60232/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 537111 T1815 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 537112 T1815 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 537116 T1815 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 537181 T1815 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 537183 T1815 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:48277
[junit4] 2> 537184 T1815 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 537184 T1815 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 537185 T1815 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383577963973
[junit4] 2> 537185 T1815 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383577963973/'
[junit4] 2> 537194 T1815 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383577963973/solr.xml
[junit4] 2> 537224 T1815 oasc.CoreContainer.<init> New CoreContainer 1132164072
[junit4] 2> 537225 T1815 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383577963973/]
[junit4] 2> 537226 T1815 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
[junit4] 2> 537226 T1815 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 537226 T1815 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 537227 T1815 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 537227 T1815 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 537227 T1815 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 537228 T1815 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 537228 T1815 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 537228 T1815 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 537229 T1815 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
[junit4] 2> 537230 T1815 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 537230 T1815 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:60232/solr
[junit4] 2> 537230 T1815 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 537231 T1815 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 537233 T1855 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5df6a6f4 name:ZooKeeperConnection Watcher:127.0.0.1:60232 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 537233 T1815 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 537236 T1815 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 537238 T1857 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e950332 name:ZooKeeperConnection Watcher:127.0.0.1:60232/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 537238 T1815 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 537243 T1815 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 538247 T1815 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48277_
[junit4] 2> 538249 T1815 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48277_
[junit4] 2> 538261 T1844 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 538262 T1837 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 538262 T1857 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 538270 T1858 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 538270 T1858 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 538271 T1858 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 538582 T1838 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 538583 T1838 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "base_url":"http://127.0.0.1:55584",
[junit4] 2> "core":"collection1",
[junit4] 2> "roles":null,
[junit4] 2> "node_name":"127.0.0.1:55584_",
[junit4] 2> "shard":"shard1",
[junit4] 2> "shard_range":null,
[junit4] 2> "shard_state":"active",
[junit4] 2> "shard_parent":null,
[junit4] 2> "collection":"control_collection",
[junit4] 2> "numShards":"1",
[junit4] 2> "core_node_name":"core_node1"}
[junit4] 2> 538589 T1838 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:48277",
[junit4] 2> "core":"collection1",
[junit4] 2> "roles":null,
[junit4] 2> "node_name":"127.0.0.1:48277_",
[junit4] 2> "shard":null,
[junit4] 2> "shard_range":null,
[junit4] 2> "shard_state":"active",
[junit4] 2> "shard_parent":null,
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"2",
[junit4] 2> "core_node_name":null}
[junit4] 2> 538589 T1838 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
[junit4] 2> 538589 T1838 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 538594 T1844 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 538594 T1857 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 538595 T1837 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 539272 T1858 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
[junit4] 2> 539272 T1858 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383577963973/collection1
[junit4] 2> 539272 T1858 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4] 2> 539274 T1858 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4] 2> 539274 T1858 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4] 2> 539276 T1858 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383577963973/collection1/'
[junit4] 2> 539277 T1858 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383577963973/collection1/lib/classes/' to classloader
[junit4] 2> 539277 T1858 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383577963973/collection1/lib/README' to classloader
[junit4] 2> 539313 T1858 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_46
[junit4] 2> 539353 T1858 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4] 2> 539355 T1858 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4] 2> 539361 T1858 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4] 2> 539729 T1858 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
[junit4] 2> 539729 T1858 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
[junit4] 2> 539730 T1858 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4] 2> 539735 T1858 oass.IndexSchema.readSchema default search field in schema is text
[junit4] 2> 539738 T1858 oass.IndexSchema.readSchema unique key field: id
[junit4] 2> 539755 T1858 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 539759 T1858 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 539762 T1858 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 539764 T1858 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 539764 T1858 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 539764 T1858 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 539766 T1858 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 539766 T1858 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 539766 T1858 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4] 2> 539766 T1858 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty1-1383577963973/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/jetty1/
[junit4] 2> 539766 T1858 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cea6f21
[junit4] 2> 539767 T1858 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/jetty1
[junit4] 2> 539767 T1858 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/jetty1/index/
[junit4] 2> 539767 T1858 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/jetty1/index' doesn't exist. Creating new index...
[junit4] 2> 539768 T1858 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/jetty1/index
[junit4] 2> 539768 T1858 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@b6a536a
[junit4] 2> 539853 T1858 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4] 2> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/jetty1/index,segFN=segments_1,generation=1}
[junit4] 2> 539854 T1858 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 539857 T1858 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
[junit4] 2> 539857 T1858 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
[junit4] 2> 539857 T1858 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4] 2> 539857 T1858 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
[junit4] 2> 539858 T1858 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4] 2> 539858 T1858 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
[junit4] 2> 539858 T1858 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 539858 T1858 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 539859 T1858 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
[junit4] 2> 539859 T1858 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4] 2> 539859 T1858 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4] 2> 539860 T1858 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4] 2> 539860 T1858 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4] 2> 539860 T1858 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4] 2> 539861 T1858 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4] 2> 539862 T1858 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4] 2> 539864 T1858 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4] 2> 539864 T1858 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4] 2> 539865 T1858 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=19, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 539866 T1858 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4] 2> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/jetty1/index,segFN=segments_1,generation=1}
[junit4] 2> 539866 T1858 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 539866 T1858 oass.SolrIndexSearcher.<init> Opening Searcher@73016bf3 main
[junit4] 2> 539869 T1859 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@73016bf3 main{StandardDirectoryReader(segments_1:1:nrt)}
[junit4] 2> 539871 T1858 oasc.CoreContainer.registerCore registering core: collection1
[junit4] 2> 539871 T1858 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:48277 collection:collection1 shard:shard1
[junit4] 2> 539872 T1858 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 539884 T1858 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
[junit4] 2> 539887 T1858 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4] 2> 539887 T1858 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4] 2> 539887 T1858 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:48277/collection1/
[junit4] 2> 539888 T1858 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4] 2> 539888 T1858 oasc.SyncStrategy.syncToMe http://127.0.0.1:48277/collection1/ has no replicas
[junit4] 2> 539888 T1858 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:48277/collection1/ shard1
[junit4] 2> 539888 T1858 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 540104 T1838 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 540114 T1844 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 540114 T1837 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 540114 T1857 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 540148 T1858 oasc.ZkController.register We are http://127.0.0.1:48277/collection1/ and leader is http://127.0.0.1:48277/collection1/
[junit4] 2> 540148 T1858 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:48277
[junit4] 2> 540148 T1858 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4] 2> 540149 T1858 oasc.ZkController.publish publishing core=collection1 state=active
[junit4] 2> 540149 T1858 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 540151 T1858 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 540152 T1815 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
[junit4] 2> 540152 T1815 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4] 2> 540255 T1815 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 540258 T1815 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:48168
[junit4] 2> 540259 T1815 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 540259 T1815 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 540259 T1815 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383577967011
[junit4] 2> 540260 T1815 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383577967011/'
[junit4] 2> 540270 T1815 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383577967011/solr.xml
[junit4] 2> 540310 T1815 oasc.CoreContainer.<init> New CoreContainer 154053078
[junit4] 2> 540310 T1815 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383577967011/]
[junit4] 2> 540311 T1815 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
[junit4] 2> 540311 T1815 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 540312 T1815 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 540312 T1815 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 540312 T1815 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 540313 T1815 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 540313 T1815 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 540313 T1815 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 540314 T1815 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 540315 T1815 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
[junit4] 2> 540315 T1815 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 540315 T1815 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:60232/solr
[junit4] 2> 540316 T1815 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 540317 T1815 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 540318 T1871 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5690254f name:ZooKeeperConnection Watcher:127.0.0.1:60232 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 540319 T1815 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 540322 T1815 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 540323 T1873 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@70b98528 name:ZooKeeperConnection Watcher:127.0.0.1:60232/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 540323 T1815 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 540329 T1815 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 541336 T1815 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48168_
[junit4] 2> 541338 T1815 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48168_
[junit4] 2> 541344 T1844 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 541345 T1837 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 541345 T1873 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 541346 T1857 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 541355 T1874 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 541355 T1874 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 541421 T1874 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 541624 T1838 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 541625 T1838 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "base_url":"http://127.0.0.1:48277",
[junit4] 2> "core":"collection1",
[junit4] 2> "roles":null,
[junit4] 2> "node_name":"127.0.0.1:48277_",
[junit4] 2> "shard":"shard1",
[junit4] 2> "shard_range":null,
[junit4] 2> "shard_state":"active",
[junit4] 2> "shard_parent":null,
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"2",
[junit4] 2> "core_node_name":"core_node1"}
[junit4] 2> 541631 T1838 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:48168",
[junit4] 2> "core":"collection1",
[junit4] 2> "roles":null,
[junit4] 2> "node_name":"127.0.0.1:48168_",
[junit4] 2> "shard":null,
[junit4] 2> "shard_range":null,
[junit4] 2> "shard_state":"active",
[junit4] 2> "shard_parent":null,
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"2",
[junit4] 2> "core_node_name":null}
[junit4] 2> 541631 T1838 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4] 2> 541631 T1838 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
[junit4] 2> 541636 T1837 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 541636 T1857 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 541636 T1844 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 541636 T1873 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 542422 T1874 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
[junit4] 2> 542422 T1874 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383577967011/collection1
[junit4] 2> 542422 T1874 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4] 2> 542423 T1874 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4] 2> 542424 T1874 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4] 2> 542425 T1874 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383577967011/collection1/'
[junit4] 2> 542426 T1874 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383577967011/collection1/lib/classes/' to classloader
[junit4] 2> 542426 T1874 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383577967011/collection1/lib/README' to classloader
[junit4] 2> 542463 T1874 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_46
[junit4] 2> 542493 T1874 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4] 2> 542495 T1874 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4] 2> 542501 T1874 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4] 2> 542847 T1874 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
[junit4] 2> 542848 T1874 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
[junit4] 2> 542848 T1874 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4] 2> 542854 T1874 oass.IndexSchema.readSchema default search field in schema is text
[junit4] 2> 542856 T1874 oass.IndexSchema.readSchema unique key field: id
[junit4] 2> 542874 T1874 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 542878 T1874 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 542881 T1874 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 542883 T1874 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 542883 T1874 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 542883 T1874 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 542884 T1874 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 542885 T1874 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 542885 T1874 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4] 2> 542885 T1874 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty2-1383577967011/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/jetty2/
[junit4] 2> 542885 T1874 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cea6f21
[junit4] 2> 542886 T1874 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/jetty2
[junit4] 2> 542886 T1874 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/jetty2/index/
[junit4] 2> 542886 T1874 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/jetty2/index' doesn't exist. Creating new index...
[junit4] 2> 542887 T1874 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/jetty2/index
[junit4] 2> 542887 T1874 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@3f61a8c0
[junit4] 2> 542889 T1874 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4] 2> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/jetty2/index,segFN=segments_1,generation=1}
[junit4] 2> 542890 T1874 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 542893 T1874 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
[junit4] 2> 542893 T1874 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
[junit4] 2> 542893 T1874 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4] 2> 542893 T1874 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
[junit4] 2> 542894 T1874 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4] 2> 542894 T1874 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
[junit4] 2> 542894 T1874 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 542894 T1874 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 542894 T1874 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
[junit4] 2> 542895 T1874 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4] 2> 542895 T1874 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4] 2> 542896 T1874 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4] 2> 542896 T1874 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4] 2> 542896 T1874 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4] 2> 542896 T1874 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4] 2> 542898 T1874 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4] 2> 542900 T1874 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4] 2> 542900 T1874 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4] 2> 542901 T1874 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=19, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 542901 T1874 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4] 2> commit{dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-1383577960424/jetty2/index,segFN=segments_1,generation=1}
[junit4] 2> 542902 T1874 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 542902 T1874 oass.SolrIndexSearcher.<init> Opening Searcher@9b0bee3 main
[junit4] 2> 542904 T1875 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@9b0bee3 main{StandardDirectoryReader(segments_1:1:nrt)}
[junit4] 2> 542906 T1874 oasc.CoreContainer.registerCore registering core: collection1
[junit4] 2> 542906 T1874 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:48168 collection:collection1 shard:shard2
[junit4] 2> 542907 T1874 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4] 2> 542916 T1874 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
[junit4] 2> 542919 T1874 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4] 2> 542920 T1874 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4] 2> 542920 T1874 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:48168/collection1/
[junit4] 2> 542920 T1874 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4] 2> 542920 T1874 oasc.SyncStrategy.syncToMe http://127.0.0.1:48168/collection1/ has no replicas
[junit4] 2> 542920 T1874 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:48168/collection1/ shard2
[junit4] 2> 542921 T1874 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4] 2> 543146 T1838 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 543157 T1837 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 543157 T1844 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 543157 T1857 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 543158 T1873 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 543179 T1874 oasc.ZkController.register We are http://127.0.0.1:48168/collection1/ and leader is http://127.0.0.1:48168/collection1/
[junit4] 2> 543179 T1874 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:48168
[junit4] 2> 543179 T1874 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4] 2> 543179 T1874 oasc.ZkController.publish publishing core=collection1 state=active
[junit4] 2> 543179 T1874 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 543181 T1874 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 543182 T1815 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
[junit4] 2> 543183 T1815 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4] 2> 543253 T1815 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 543255 T1815 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:47760
[junit4] 2> 543256 T1815 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 543256 T1815 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 543257 T1815 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty3-1383577970040
[junit4] 2> 543257 T1815 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty3-1383577970040/'
[junit4] 2> 543266 T1815 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty3-1383577970040/solr.xml
[junit4] 2> 543295 T1815 oasc.CoreContainer.<init> New CoreContainer 239735805
[junit4] 2> 543296 T1815 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest-jetty3-1383577970040/]
[junit4] 2> 543297 T1815 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
[junit4] 2> 543297 T1815 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 543297 T1815 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 543297 T1815 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 543298 T1815 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 543298 T1815 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 543298 T1815 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 543299 T1815 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 543299 T1815 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 543300 T1815 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
[junit4] 2> 543300 T1815 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 543301 T1815 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:60232/solr
[junit4] 2> 543301 T1815 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 543302 T1815 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 543304 T1887 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21664e5f name:ZooKeeperConnection Watcher:127.0.0.1:60232 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 543304 T1815 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 543307 T1815 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 543308 T1889 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16b374ce name:ZooKeeperConnection Watcher:127.0.0.1:60232/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 543309 T1815 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 543314 T1815 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 544319 T1815 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47760_
[junit4] 2> 544320 T1815 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:47760_
[junit4] 2> 544326 T1837 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 544326 T1857 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 544326 T1889 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 544327 T1844 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 544328 T1873 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 544334 T1890 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 544334 T1890 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 544335 T1890 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 544667 T1838 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 544668 T1838 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "base_url":"http://127.0.0.1:48168",
[junit4] 2> "core":"collection1",
[junit4] 2> "roles":null,
[junit4] 2> "node_name":"127.0.0.1:48168_",
[junit4] 2> "shard":"shard2",
[junit4] 2> "shard_range":null,
[junit4] 2> "shard_state":"active",
[junit4] 2> "shard_parent":null,
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"2",
[junit4] 2> "core_node_name":"core_node2"}
[junit4] 2> 544674 T1838 oasc.Overseer$ClusterStateUpdater.update
[...truncated too long message...]
> at java.util.TimerThread.mainLoop(Timer.java:509)
[junit4] > at java.util.TimerThread.run(Timer.java:462)
[junit4] > 5) Thread[id=2031, name=qtp1642456440-2031, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
[junit4] > at java.lang.Thread.run(Thread.java:662)
[junit4] > 6) Thread[id=2032, name=qtp1642456440-2032, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
[junit4] > at java.lang.Thread.run(Thread.java:662)
[junit4] > 7) Thread[id=2029, name=qtp1642456440-2029 Acceptor1 SelectChannelConnector@127.0.0.1:48168, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:134)
[junit4] > at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
[junit4] > at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:662)
[junit4] > 8) Thread[id=2030, name=qtp1642456440-2030, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
[junit4] > at java.lang.Thread.run(Thread.java:662)
[junit4] > 9) Thread[id=2033, name=qtp1642456440-2033, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
[junit4] > at java.lang.Thread.run(Thread.java:662)
[junit4] > at __randomizedtesting.SeedInfo.seed([C13BD390334B6218]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=2027, name=qtp1642456440-2027 Selector1, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
[junit4] > at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
[junit4] > at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:662)
[junit4] > 2) Thread[id=2026, name=qtp1642456440-2026 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
[junit4] > at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
[junit4] > at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:662)
[junit4] > 3) Thread[id=2028, name=qtp1642456440-2028 Acceptor0 SelectChannelConnector@127.0.0.1:48168, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:956)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:662)
[junit4] > 4) Thread[id=2025, name=HashSessionScavenger-65, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.util.TimerThread.mainLoop(Timer.java:509)
[junit4] > at java.util.TimerThread.run(Timer.java:462)
[junit4] > 5) Thread[id=2029, name=qtp1642456440-2029 Acceptor1 SelectChannelConnector@127.0.0.1:48168, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:102)
[junit4] > at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:662)
[junit4] > at __randomizedtesting.SeedInfo.seed([C13BD390334B6218]:0)
[junit4] Completed on J1 in 159.29s, 1 test, 2 errors <<< FAILURES!
[...truncated 566 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:428: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:408: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:483: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1263: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:906: There were test failures: 333 suites, 1469 tests, 2 suite-level errors, 701 ignored (5 assumptions)
Total time: 35 minutes 19 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.6.0_45 -XX:+UseCompressedOops -XX:+UseSerialGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure