You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2019/12/07 15:19:00 UTC
[JENKINS] Lucene-Solr-BadApples-NightlyTests-8.x - Build # 46 -
Still Unstable
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-8.x/46/
11 tests failed.
FAILED: org.apache.lucene.codecs.asserting.TestAssertingDocValuesFormat.testSparseLongNumericsVsStoredFields
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([D446FDC9F8B4342D]:0)
FAILED: junit.framework.TestSuite.org.apache.lucene.codecs.asserting.TestAssertingDocValuesFormat
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([D446FDC9F8B4342D]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
Error Message:
access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
at __randomizedtesting.SeedInfo.seed([D1C21E02EF7F767C]:0)
at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
at java.security.AccessController.checkPermission(AccessController.java:884)
at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
at org.apache.hadoop.util.Shell.run(Shell.java:901)
at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74)
at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.<init>(NameNodeResourceChecker.java:70)
at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166)
at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.<init>(NameNodeResourceChecker.java:135)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168)
at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791)
at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717)
at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:940)
at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:913)
at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646)
at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314)
at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083)
at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
at org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.setupClass(HdfsUnloadDistributedZkTest.java:39)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
Error Message:
15 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest: 1) Thread[id=3399, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=3406, name=qtp1114988144-3406, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=3417, name=IPC Server idle connection scanner for port 38153, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) at java.util.TimerThread.run(Timer.java:505) 4) Thread[id=3407, name=qtp1114988144-3407, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=3397, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 6) Thread[id=3414, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221) at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=3416, name=Socket Reader #1 for port 38153, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) 8) Thread[id=3402, name=qtp1114988144-3402-acceptor-0@300e6f1d-ServerConnector@40de12fc{HTTP/1.1,[http/1.1]}{lucene2-us-west.apache.org:36343}, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=3405, name=qtp1114988144-3405, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=3401, name=qtp1114988144-3401, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=3403, name=qtp1114988144-3403, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=3400, name=qtp1114988144-3400, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=3404, name=qtp1114988144-3404, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=3408, name=Session-HouseKeeper-5161250c, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=3398, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@24b30f80, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest:
1) Thread[id=3399, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=3406, name=qtp1114988144-3406, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=3417, name=IPC Server idle connection scanner for port 38153, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
at java.util.TimerThread.run(Timer.java:505)
4) Thread[id=3407, name=qtp1114988144-3407, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=3397, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
6) Thread[id=3414, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)
at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=3416, name=Socket Reader #1 for port 38153, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
8) Thread[id=3402, name=qtp1114988144-3402-acceptor-0@300e6f1d-ServerConnector@40de12fc{HTTP/1.1,[http/1.1]}{lucene2-us-west.apache.org:36343}, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=3405, name=qtp1114988144-3405, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=3401, name=qtp1114988144-3401, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=3403, name=qtp1114988144-3403, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=3400, name=qtp1114988144-3400, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=3404, name=qtp1114988144-3404, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=3408, name=Session-HouseKeeper-5161250c, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=3398, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@24b30f80, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([D1C21E02EF7F767C]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=3399, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=3406, name=qtp1114988144-3406, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=3417, name=IPC Server idle connection scanner for port 38153, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) at java.util.TimerThread.run(Timer.java:505) 4) Thread[id=3407, name=qtp1114988144-3407, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=3397, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 6) Thread[id=3416, name=Socket Reader #1 for port 38153, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) 7) Thread[id=3402, name=qtp1114988144-3402, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=3405, name=qtp1114988144-3405, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=3401, name=qtp1114988144-3401, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=3403, name=qtp1114988144-3403, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=3400, name=qtp1114988144-3400, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=3404, name=qtp1114988144-3404, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=3408, name=Session-HouseKeeper-5161250c, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=3399, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=3406, name=qtp1114988144-3406, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=3417, name=IPC Server idle connection scanner for port 38153, state=WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
at java.util.TimerThread.run(Timer.java:505)
4) Thread[id=3407, name=qtp1114988144-3407, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=3397, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
6) Thread[id=3416, name=Socket Reader #1 for port 38153, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.IOUtil.drain(Native Method)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
7) Thread[id=3402, name=qtp1114988144-3402, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=3405, name=qtp1114988144-3405, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=3401, name=qtp1114988144-3401, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=3403, name=qtp1114988144-3403, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=3400, name=qtp1114988144-3400, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$35/12111668.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=3404, name=qtp1114988144-3404, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=3408, name=Session-HouseKeeper-5161250c, state=TIMED_WAITING, group=TGRP-HdfsUnloadDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([D1C21E02EF7F767C]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestHdfsBackupRestoreCore
Error Message:
access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "<<ALL FILES>>" "execute")
at __randomizedtesting.SeedInfo.seed([D1C21E02EF7F767C]:0)
at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
at java.security.AccessController.checkPermission(AccessController.java:884)
at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
at java.lang.SecurityManager.checkExec(SecurityManager.java:799)
at java.lang.ProcessBuilder.start(ProcessBuilder.java:1018)
at org.apache.hadoop.util.Shell.runCommand(Shell.java:938)
at org.apache.hadoop.util.Shell.run(Shell.java:901)
at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74)
at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.<init>(NameNodeResourceChecker.java:70)
at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166)
at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.<init>(NameNodeResourceChecker.java:135)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168)
at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791)
at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717)
at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:940)
at org.apache.hadoop.hdfs.server.namenode.NameNode.<init>(NameNode.java:913)
at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646)
at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314)
at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083)
at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958)
at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890)
at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:518)
at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477)
at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134)
at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66)
at org.apache.solr.handler.TestHdfsBackupRestoreCore.setupClass(TestHdfsBackupRestoreCore.java:106)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestHdfsBackupRestoreCore
Error Message:
15 threads leaked from SUITE scope at org.apache.solr.handler.TestHdfsBackupRestoreCore: 1) Thread[id=67, name=IPC Server idle connection scanner for port 39612, state=WAITING, group=TGRP-TestHdfsBackupRestoreCore] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) at java.util.TimerThread.run(Timer.java:505) 2) Thread[id=52, name=qtp1869227482-52, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/155039732.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=49, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=58, name=Session-HouseKeeper-6b72f723, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=50, name=qtp1869227482-50, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/155039732.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=51, name=qtp1869227482-51, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=48, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@275cc6db, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=66, name=Socket Reader #1 for port 39612, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) 9) Thread[id=53, name=qtp1869227482-53-acceptor-0@28dabb44-ServerConnector@1304731f{HTTP/1.1,[http/1.1]}{lucene2-us-west.apache.org:45735}, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=57, name=qtp1869227482-57, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=55, name=qtp1869227482-55, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=47, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 13) Thread[id=56, name=qtp1869227482-56, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=64, name=FSEditLogAsync, state=WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221) at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=54, name=qtp1869227482-54, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.handler.TestHdfsBackupRestoreCore:
1) Thread[id=67, name=IPC Server idle connection scanner for port 39612, state=WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
at java.util.TimerThread.run(Timer.java:505)
2) Thread[id=52, name=qtp1869227482-52, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/155039732.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=49, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=58, name=Session-HouseKeeper-6b72f723, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=50, name=qtp1869227482-50, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/155039732.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=51, name=qtp1869227482-51, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=48, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@275cc6db, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at java.lang.Thread.sleep(Native Method)
at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=66, name=Socket Reader #1 for port 39612, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
9) Thread[id=53, name=qtp1869227482-53-acceptor-0@28dabb44-ServerConnector@1304731f{HTTP/1.1,[http/1.1]}{lucene2-us-west.apache.org:45735}, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=57, name=qtp1869227482-57, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=55, name=qtp1869227482-55, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=47, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
13) Thread[id=56, name=qtp1869227482-56, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=64, name=FSEditLogAsync, state=WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221)
at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=54, name=qtp1869227482-54, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([D1C21E02EF7F767C]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestHdfsBackupRestoreCore
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=67, name=IPC Server idle connection scanner for port 39612, state=WAITING, group=TGRP-TestHdfsBackupRestoreCore] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) at java.util.TimerThread.run(Timer.java:505) 2) Thread[id=52, name=qtp1869227482-52, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/155039732.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=49, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=58, name=Session-HouseKeeper-6b72f723, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=50, name=qtp1869227482-50, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/155039732.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=51, name=qtp1869227482-51, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=66, name=Socket Reader #1 for port 39612, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) 8) Thread[id=53, name=qtp1869227482-53, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=57, name=qtp1869227482-57, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=55, name=qtp1869227482-55, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=47, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 12) Thread[id=56, name=qtp1869227482-56, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=54, name=qtp1869227482-54, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=67, name=IPC Server idle connection scanner for port 39612, state=WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
at java.util.TimerThread.run(Timer.java:505)
2) Thread[id=52, name=qtp1869227482-52, state=RUNNABLE, group=TGRP-TestHdfsBackupRestoreCore]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/155039732.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=49, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsBackupRestoreCore]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueue
[...truncated too long message...]
jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=2683, name=qtp108829400-2683, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=2676, name=qtp108829400-2676, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1497764942.run(Unknown Source)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=2684, name=Session-HouseKeeper-5b21ba47, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 8) Thread[id=2678, name=qtp108829400-2678, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 9) Thread[id=2692, name=Socket Reader #1 for port 42757, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
[junit4] > at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
[junit4] > at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
[junit4] > at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097)
[junit4] > at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076)
[junit4] > 10) Thread[id=2675, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 11) Thread[id=2680, name=qtp108829400-2680, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 12) Thread[id=2677, name=qtp108829400-2677, state=RUNNABLE, group=TGRP-HdfsDirectoryTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$$Lambda$59/1497764942.run(Unknown Source)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 13) Thread[id=2693, name=IPC Server idle connection scanner for port 42757, state=WAITING, group=TGRP-HdfsDirectoryTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at java.util.TimerThread.mainLoop(Timer.java:526)
[junit4] > at java.util.TimerThread.run(Timer.java:505)
[junit4] > at __randomizedtesting.SeedInfo.seed([D1C21E02EF7F767C]:0)
[junit4] Completed [746/887 (3!)] on J1 in 15.63s, 0 tests, 3 errors <<< FAILURES!
[...truncated 53349 lines...]
[JENKINS] Lucene-Solr-BadApples-NightlyTests-8.x - Build # 49 -
Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-8.x/49/
1 tests failed.
FAILED: org.apache.lucene.document.TestXYMultiPolygonShapeQueries.testRandomBig
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([4B8813CF298D533E:CCDF6E40B8D42FBE]:0)
at org.apache.lucene.util.bkd.HeapPointWriter.<init>(HeapPointWriter.java:42)
at org.apache.lucene.util.bkd.BKDRadixSelector.getPointWriter(BKDRadixSelector.java:513)
at org.apache.lucene.util.bkd.BKDRadixSelector.select(BKDRadixSelector.java:123)
at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1647)
at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1669)
at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1669)
at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
at org.apache.lucene.util.bkd.BKDWriter.finish(BKDWriter.java:797)
at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:140)
at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:191)
at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:143)
at org.apache.lucene.codecs.asserting.AssertingPointsFormat$AssertingPointsWriter.merge(AssertingPointsFormat.java:149)
at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:202)
at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:162)
at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2158)
at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5120)
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1597)
at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
at org.apache.lucene.document.BaseShapeTestCase.indexRandomShapes(BaseShapeTestCase.java:231)
at org.apache.lucene.document.BaseShapeTestCase.verify(BaseShapeTestCase.java:212)
at org.apache.lucene.document.BaseShapeTestCase.doTestRandom(BaseShapeTestCase.java:138)
at org.apache.lucene.document.TestXYMultiPolygonShapeQueries.testRandomBig(TestXYMultiPolygonShapeQueries.java:143)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
Build Log:
[...truncated 11356 lines...]
[junit4] Suite: org.apache.lucene.document.TestXYMultiPolygonShapeQueries
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestXYMultiPolygonShapeQueries -Dtests.method=testRandomBig -Dtests.seed=4B8813CF298D533E -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=ro-RO -Dtests.timezone=Asia/Famagusta -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 973s J1 | TestXYMultiPolygonShapeQueries.testRandomBig <<<
[junit4] > Throwable #1: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([4B8813CF298D533E:CCDF6E40B8D42FBE]:0)
[junit4] > at org.apache.lucene.util.bkd.HeapPointWriter.<init>(HeapPointWriter.java:42)
[junit4] > at org.apache.lucene.util.bkd.BKDRadixSelector.getPointWriter(BKDRadixSelector.java:513)
[junit4] > at org.apache.lucene.util.bkd.BKDRadixSelector.select(BKDRadixSelector.java:123)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1647)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1669)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1669)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1664)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.finish(BKDWriter.java:797)
[junit4] > at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:140)
[junit4] > at org.apache.lucene.codecs.PointsWriter.mergeOneField(PointsWriter.java:62)
[junit4] > at org.apache.lucene.codecs.PointsWriter.merge(PointsWriter.java:191)
[junit4] > at org.apache.lucene.codecs.lucene60.Lucene60PointsWriter.merge(Lucene60PointsWriter.java:143)
[junit4] > at org.apache.lucene.codecs.asserting.AssertingPointsFormat$AssertingPointsWriter.merge(AssertingPointsFormat.java:149)
[junit4] > at org.apache.lucene.index.SegmentMerger.mergePoints(SegmentMerger.java:202)
[junit4] > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:162)
[junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
[junit4] > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
[junit4] > at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
[junit4] > at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2158)
[junit4] > at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5120)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1597)
[junit4] > at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
[junit4] > at org.apache.lucene.document.BaseShapeTestCase.indexRandomShapes(BaseShapeTestCase.java:231)
[junit4] > at org.apache.lucene.document.BaseShapeTestCase.verify(BaseShapeTestCase.java:212)
[junit4] > at org.apache.lucene.document.BaseShapeTestCase.doTestRandom(BaseShapeTestCase.java:138)
[junit4] > at org.apache.lucene.document.TestXYMultiPolygonShapeQueries.testRandomBig(TestXYMultiPolygonShapeQueries.java:143)
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/sandbox/test/J1/temp/lucene.document.TestXYMultiPolygonShapeQueries_4B8813CF298D533E-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene84): {id=BlockTreeOrds(blocksize=128)}, docValues:{shape=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=928, maxMBSortInHeap=6.872291520847615, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@309ed5de), locale=ro-RO, timezone=Asia/Famagusta
[junit4] 2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=151691928,total=288882688
[junit4] 2> NOTE: All tests run in this JVM: [TestBigIntegerPoint, TestXYMultiPolygonShapeQueries]
[junit4] Completed [35/40 (1!)] on J1 in 1754.89s, 5 tests, 1 error <<< FAILURES!
[...truncated 18 lines...]
[junit4] JVM J1: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/sandbox/test/temp/junit4-J1-20191227_194341_1611762379036369179465.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/heapdumps/java_pid20764.hprof ...
[junit4] Heap dump file created [451379878 bytes in 0.773 secs]
[junit4] <<< JVM J1: EOF ----
[...truncated 15185 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/build.xml:662: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/build.xml:586: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid20764.hprof
Total time: 506 minutes 55 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-BadApples-NightlyTests-8.x - Build # 48 -
Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-8.x/48/
1 tests failed.
FAILED: org.apache.lucene.search.TestDoubleRangeFieldQueries.testRandomBig
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([4E8B4ACB3E2B5857:C9DC3744AF7224D7]:0)
at org.apache.lucene.util.ArrayUtil.growExact(ArrayUtil.java:302)
at org.apache.lucene.codecs.compressing.CompressingStoredFieldsIndexReader.<init>(CompressingStoredFieldsIndexReader.java:75)
at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.<init>(CompressingTermVectorsReader.java:128)
at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsReader(CompressingTermVectorsFormat.java:91)
at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:130)
at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:679)
at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:302)
at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:862)
at org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
at org.apache.lucene.util.IOUtils.close(IOUtils.java:77)
at org.apache.lucene.search.BaseRangeFieldQueryTestCase.verify(BaseRangeFieldQueryTestCase.java:322)
at org.apache.lucene.search.BaseRangeFieldQueryTestCase.doTestRandom(BaseRangeFieldQueryTestCase.java:189)
at org.apache.lucene.search.BaseRangeFieldQueryTestCase.testRandomBig(BaseRangeFieldQueryTestCase.java:77)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
Build Log:
[...truncated 1836 lines...]
[junit4] Suite: org.apache.lucene.search.TestDoubleRangeFieldQueries
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestDoubleRangeFieldQueries -Dtests.method=testRandomBig -Dtests.seed=4E8B4ACB3E2B5857 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=ga-IE -Dtests.timezone=Pacific/Enderbury -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 130s J0 | TestDoubleRangeFieldQueries.testRandomBig <<<
[junit4] > Throwable #1: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([4E8B4ACB3E2B5857:C9DC3744AF7224D7]:0)
[junit4] > at org.apache.lucene.util.ArrayUtil.growExact(ArrayUtil.java:302)
[junit4] > at org.apache.lucene.codecs.compressing.CompressingStoredFieldsIndexReader.<init>(CompressingStoredFieldsIndexReader.java:75)
[junit4] > at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.<init>(CompressingTermVectorsReader.java:128)
[junit4] > at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsReader(CompressingTermVectorsFormat.java:91)
[junit4] > at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:130)
[junit4] > at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
[junit4] > at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:679)
[junit4] > at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:302)
[junit4] > at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:862)
[junit4] > at org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
[junit4] > at org.apache.lucene.util.IOUtils.close(IOUtils.java:77)
[junit4] > at org.apache.lucene.search.BaseRangeFieldQueryTestCase.verify(BaseRangeFieldQueryTestCase.java:322)
[junit4] > at org.apache.lucene.search.BaseRangeFieldQueryTestCase.doTestRandom(BaseRangeFieldQueryTestCase.java:189)
[junit4] > at org.apache.lucene.search.BaseRangeFieldQueryTestCase.testRandomBig(BaseRangeFieldQueryTestCase.java:77)
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/core/test/J0/temp/lucene.search.TestDoubleRangeFieldQueries_4E8B4ACB3E2B5857-001
[junit4] 2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=4, maxDocsPerChunk=608, blockSize=1), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=4, blockSize=1)), sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@8646d33), locale=ga-IE, timezone=Pacific/Enderbury
[junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=243183856,total=327155712
[junit4] 2> NOTE: All tests run in this JVM: [TestLazyProxSkipping, TestTieredMergePolicy, TestLucene50StoredFieldsFormatHighCompression, TestIntsRef, TestTermVectorsWriter, TestCharTermAttributeImpl, TestMultiPhraseQuery, TestNeedsScores, TestSimpleFSDirectory, TestEarlyTermination, TestSmallFloat, TestLucene80NormsFormatMergeInstance, TestInfoStream, TestLongValuesSource, TestRegExp, TestFieldInvertState, TestLucene60FieldInfoFormat, TestAtomicUpdate, TestManyFields, TestIndexWriterReader, TestFutureObjects, TestMergeSchedulerExternal, TestPriorityQueue, TestLucene84PostingsFormat, TestAxiomaticF2LOG, TestSearcherManager, TestBlockMaxConjunction, TestNRTThreads, TestSimilarity, TestSpanBoostQuery, TestNoDeletionPolicy, TestDocIdsWriter, TestMergeRateLimiter, TestIndexWriterMerging, TestSegmentInfos, TestPolygon, TestIndexWriterLockRelease, TestSimilarity2, TestSimpleExplanationsOfNonMatches, TestCharArraySet, Test4GBStoredFields, TestIntRange, TestTransactions, TestSortedSetDocValues, TestBKD, TestBoostQuery, TestRollback, TestStressIndexing, TestTimSorterWorstCase, TestPerSegmentDeletes, TestBooleanQueryVisitSubscorers, TestReqOptSumScorer, TestMultiDocValues, TestDirectoryReaderReopen, TestFuzzyQuery, TestIndexWriterExceptions2, TestBytesRefHash, TestIndependenceStandardized, TestNumericUtils, TestNativeFSLockFactory, TestDocumentsWriterStallControl, TestMultiTermsEnum, TestBinaryDocValuesUpdates, TestComplexExplanationsOfNonMatches, TestSpanExplanations, TestIntArrayDocIdSet, TestBytesStore, TestSynonymQuery, TestAllFilesDetectTruncation, TestFrozenBufferedUpdates, TestTotalHitCountCollector, TestLongsRef, TestDisjunctionScoreBlockBoundaryPropagator, TestDaciukMihovAutomatonBuilder, TestLucene50CompoundFormat, TestExceedMaxTermLength, TestBooleanSimilarity, TestQueryVisitor, TestPrefixQuery, TestDeletionPolicy, TestTopDocsMerge, TestMinShouldMatch2, TestReusableStringReader, TestIndexWriterDelete, TestSpanMultiTermQueryWrapper, TestFileSwitchDirectory, Test2BPostingsBytes, Test2BSortedDocValuesFixedSorted, Test2BSortedDocValuesOrds, Test2BTerms, TestCheckIndex, TestCodecHoldsOpenFiles, TestCodecs, TestConcurrentMergeScheduler, TestConsistentFieldNumbers, TestCrash, TestCrashCausesCorruptIndex, TestCustomNorms, TestCustomTermFreq, TestDocIDMerger, TestDocInverterPerFieldErrorInfo, TestDocValues, TestDocValuesFieldUpdates, TestDocValuesIndexing, TestDocsAndPositions, TestDocsWithFieldSet, TestDocumentWriter, TestDocumentsWriterDeleteQueue, TestFieldsReader, TestFilterCodecReader, TestFilterDirectoryReader, TestFilterLeafReader, TestFilterMergePolicy, TestFlex, TestFlushByRamOrCountsPolicy, TestForTooMuchCloning, TestForceMergeForever, TestIndexInput, TestIndexManyDocuments, TestIndexOptions, TestIndexReaderClose, TestIndexSorting, TestIndexTooManyDocs, TestIndexWriter, TestIndexWriterCommit, TestIndexWriterConfig, TestIndexWriterOnJRECrash, TestIndexWriterOnVMError, TestDoubleRangeFieldQueries]
[junit4] Completed [409/528 (1!)] on J0 in 136.96s, 7 tests, 1 error <<< FAILURES!
[...truncated 396 lines...]
[junit4] JVM J0: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/core/test/temp/junit4-J0-20191220_154259_8788462022269486629784.sysout
[junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/heapdumps/java_pid20293.hprof ...
[junit4] Heap dump file created [522488851 bytes in 2.500 secs]
[junit4] <<< JVM J0: EOF ----
[...truncated 24030 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/build.xml:662: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/build.xml:586: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid20293.hprof
Total time: 316 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-BadApples-NightlyTests-8.x - Build # 47 -
Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-8.x/47/
7 tests failed.
FAILED: org.apache.lucene.replicator.nrt.TestStressNRTReplication.test
Error Message:
Captured an uncaught exception in thread: Thread[id=19, name=restarter, state=RUNNABLE, group=TGRP-TestStressNRTReplication]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=19, name=restarter, state=RUNNABLE, group=TGRP-TestStressNRTReplication]
at __randomizedtesting.SeedInfo.seed([AB64A8554B0B5F2A:2330978FE5F732D2]:0)
Caused by: java.lang.RuntimeException: java.lang.AssertionError: newPrimary.initCommitVersion=3 is missing from versionToTransLogLocation: keys=[0, 65, 2, 67, 4, 37, 11, 43, 16, 48, 20, 56, 27, 31, 63]
at __randomizedtesting.SeedInfo.seed([AB64A8554B0B5F2A]:0)
at org.apache.lucene.replicator.nrt.TestStressNRTReplication$RestartThread.run(TestStressNRTReplication.java:885)
Caused by: java.lang.AssertionError: newPrimary.initCommitVersion=3 is missing from versionToTransLogLocation: keys=[0, 65, 2, 67, 4, 37, 11, 43, 16, 48, 20, 56, 27, 31, 63]
at org.apache.lucene.replicator.nrt.TestStressNRTReplication.startPrimary(TestStressNRTReplication.java:451)
at org.apache.lucene.replicator.nrt.TestStressNRTReplication.promoteReplica(TestStressNRTReplication.java:423)
at org.apache.lucene.replicator.nrt.TestStressNRTReplication.access$500(TestStressNRTReplication.java:117)
at org.apache.lucene.replicator.nrt.TestStressNRTReplication$RestartThread.run(TestStressNRTReplication.java:830)
FAILED: junit.framework.TestSuite.org.apache.lucene.replicator.nrt.TestStressNRTReplication
Error Message:
1 thread leaked from SUITE scope at org.apache.lucene.replicator.nrt.TestStressNRTReplication: 1) Thread[id=113, name=pump0, state=RUNNABLE, group=TGRP-TestStressNRTReplication] at java.io.FileInputStream.readBytes(Native Method) at java.io.FileInputStream.read(FileInputStream.java:255) at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.io.BufferedInputStream.read(BufferedInputStream.java:345) at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284) at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326) at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178) at java.io.InputStreamReader.read(InputStreamReader.java:184) at java.io.BufferedReader.fill(BufferedReader.java:161) at java.io.BufferedReader.readLine(BufferedReader.java:324) at java.io.BufferedReader.readLine(BufferedReader.java:389) at org.apache.lucene.replicator.nrt.ThreadPumper$1.run(ThreadPumper.java:37)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.lucene.replicator.nrt.TestStressNRTReplication:
1) Thread[id=113, name=pump0, state=RUNNABLE, group=TGRP-TestStressNRTReplication]
at java.io.FileInputStream.readBytes(Native Method)
at java.io.FileInputStream.read(FileInputStream.java:255)
at java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
at java.io.BufferedInputStream.read(BufferedInputStream.java:345)
at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284)
at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326)
at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178)
at java.io.InputStreamReader.read(InputStreamReader.java:184)
at java.io.BufferedReader.fill(BufferedReader.java:161)
at java.io.BufferedReader.readLine(BufferedReader.java:324)
at java.io.BufferedReader.readLine(BufferedReader.java:389)
at org.apache.lucene.replicator.nrt.ThreadPumper$1.run(ThreadPumper.java:37)
at __randomizedtesting.SeedInfo.seed([AB64A8554B0B5F2A]:0)
FAILED: junit.framework.TestSuite.org.apache.lucene.replicator.nrt.TestStressNRTReplication
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=113, name=pump0, state=RUNNABLE, group=TGRP-TestStressNRTReplication] at java.io.FileInputStream.readBytes(Native Method) at java.io.FileInputStream.read(FileInputStream.java:255) at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.io.BufferedInputStream.read(BufferedInputStream.java:345) at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284) at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326) at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178) at java.io.InputStreamReader.read(InputStreamReader.java:184) at java.io.BufferedReader.fill(BufferedReader.java:161) at java.io.BufferedReader.readLine(BufferedReader.java:324) at java.io.BufferedReader.readLine(BufferedReader.java:389) at org.apache.lucene.replicator.nrt.ThreadPumper$1.run(ThreadPumper.java:37)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=113, name=pump0, state=RUNNABLE, group=TGRP-TestStressNRTReplication]
at java.io.FileInputStream.readBytes(Native Method)
at java.io.FileInputStream.read(FileInputStream.java:255)
at java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
at java.io.BufferedInputStream.read(BufferedInputStream.java:345)
at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284)
at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326)
at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178)
at java.io.InputStreamReader.read(InputStreamReader.java:184)
at java.io.BufferedReader.fill(BufferedReader.java:161)
at java.io.BufferedReader.readLine(BufferedReader.java:324)
at java.io.BufferedReader.readLine(BufferedReader.java:389)
at org.apache.lucene.replicator.nrt.ThreadPumper$1.run(ThreadPumper.java:37)
at __randomizedtesting.SeedInfo.seed([AB64A8554B0B5F2A]:0)
FAILED: org.apache.lucene.document.TestXYMultiPolygonShapeQueries.testRandomBig
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([BBA82E75B688EE57:3CFF53FA27D192D7]:0)
at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:84)
at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:57)
at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:168)
at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:154)
at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141)
at org.apache.lucene.util.bkd.OfflinePointWriter.append(OfflinePointWriter.java:56)
at org.apache.lucene.util.bkd.BKDWriter.add(BKDWriter.java:247)
at org.apache.lucene.index.RandomCodec$1$1$1.visit(RandomCodec.java:129)
at org.apache.lucene.codecs.PointsWriter$1$1$1.visit(PointsWriter.java:117)
at org.apache.lucene.index.AssertingLeafReader$AssertingIntersectVisitor.visit(AssertingLeafReader.java:1171)
at org.apache.lucene.util.bkd.BKDReader.visitCompressedDocValues(BKDReader.java:735)
at org.apache.lucene.util.bkd.BKDReader.visitDocValuesWithCardinality(BKDReader.java:684)
at org.apache.lucene.util.bkd.BKDReader.visitDocValues(BKDReader.java:590)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:790)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:813)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:823)
at org.apache.lucene.util.bkd.BKDReader.intersect(BKDReader.java:506)
at org.apache.lucene.index.AssertingLeafReader$AssertingPointValues.intersect(AssertingLeafReader.java:1052)
at org.apache.lucene.codecs.PointsWriter$1$1.intersect(PointsWriter.java:105)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest
Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest: 1) Thread[id=1472, name=Session-HouseKeeper-4d27248, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1465, name=qtp1815760705-1465, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=1568, name=Connector-Scheduler-6805c61e, state=WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=1471, name=qtp1815760705-1471, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=1468, name=qtp1815760705-1468, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=1466, name=qtp1815760705-1466-acceptor-0@6565cb26-ServerConnector@6805c61e{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:46340}, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=1470, name=qtp1815760705-1470, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=1469, name=qtp1815760705-1469, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=1464, name=qtp1815760705-1464, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=1467, name=qtp1815760705-1467, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.api.collections.ShardSplitTest:
1) Thread[id=1472, name=Session-HouseKeeper-4d27248, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=1465, name=qtp1815760705-1465, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=1568, name=Connector-Scheduler-6805c61e, state=WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=1471, name=qtp1815760705-1471, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=1468, name=qtp1815760705-1468, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=1466, name=qtp1815760705-1466-acceptor-0@6565cb26-ServerConnector@6805c61e{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:46340}, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=1470, name=qtp1815760705-1470, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=1469, name=qtp1815760705-1469, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=1464, name=qtp1815760705-1464, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=1467, name=qtp1815760705-1467, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([97B926D1D108389F]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.api.collections.ShardSplitTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=1472, name=Session-HouseKeeper-4d27248, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1465, name=qtp1815760705-1465, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=1568, name=Connector-Scheduler-6805c61e, state=WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=1471, name=qtp1815760705-1471, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=1468, name=qtp1815760705-1468, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=1466, name=qtp1815760705-1466, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=1470, name=qtp1815760705-1470, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=1469, name=qtp1815760705-1469, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=1464, name=qtp1815760705-1464, state=RUNNABLE, group=TGRP-ShardSplitTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=1467, name=qtp1815760705-1467, state=TIMED_WAITING, group=TGRP-ShardSplitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=1472, name=Session-HouseKeeper-4d27248, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=1465, name=qtp1815760705-1465, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=1568, name=Connector-Scheduler-6805c61e, state=WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=1471, name=qtp1815760705-1471, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=1468, name=qtp1815760705-1468, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=1466, name=qtp1815760705-1466, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=1470, name=qtp1815760705-1470, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=1469, name=qtp1815760705-1469, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=1464, name=qtp1815760705-1464, state=RUNNABLE, group=TGRP-ShardSplitTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=1467, name=qtp1815760705-1467, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([97B926D1D108389F]:0)
FAILED: org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey
Error Message:
Address already in use
Stack Trace:
java.net.BindException: Address already in use
at __randomizedtesting.SeedInfo.seed([97B926D1D108389F:1C9EF500900E931B]:0)
at sun.nio.ch.Net.bind0(Native Method)
at sun.nio.ch.Net.bind(Net.java:433)
at sun.nio.ch.Net.bind(Net.java:425)
at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
at org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:342)
at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:308)
at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:236)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.eclipse.jetty.server.Server.doStart(Server.java:396)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:567)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:506)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:474)
at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:499)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1082)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1054)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11019 lines...]
[junit4] Suite: org.apache.lucene.replicator.nrt.TestStressNRTReplication
[junit4] 1> 0.001s : parent [TEST-TestStressNRTReplication.test-seed#[AB64A8554B0B5F2A]] change thread name from TEST-TestStressNRTReplication.test-seed#[AB64A8554B0B5F2A]
[junit4] 1> TEST: using 10 nodes
[junit4] 1> TEST: launch 2 indexer threads
[junit4] 1> TEST: launch 2 searcher threads
[junit4] 1> 0.072s : parent [ indexer0] top: indexer: updatePct=0.5014304102583915 sleepChance=59
[junit4] 1> 0.077s : parent [ indexer1] top: indexer: updatePct=0.5014304102583915 sleepChance=59
[junit4] 1> TEST: will run for 448 sec
[junit4] 1> 0.405s : parent [ restarter] node status x0(0.4s) x1(0.4s) x2(0.4s) x3(0.4s) x4(0.4s) x5(0.4s) x6(0.4s) x7(0.4s) x8(0.4s) x9(0.4s)
[junit4] 1> 0.405s : parent [ restarter] downNodes=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 0.406s : parent [ restarter] top: primary crashed; now pick replica to promote
[junit4] 1> 0.406s : parent [ restarter] top: no replicas running; skipping primary promotion
[junit4] 1> 0.406s : parent [ restarter] N0: top: cold start as primary
[junit4] 1> 0.406s : parent [ restarter] 0: top: startPrimary lastPrimaryVersion=0
[junit4] 1> 0.550s : parent [ restarter] JUnit version 4.12
[junit4] 1> 0.851s : parent [ main] PG=X 0 (of 10) nodes running:
[junit4] 1> 1.065s : parent [ main] PG=X 0 (of 10) nodes running:
[junit4] 1> 2.157s : parent [ restarter] .
[junit4] 1> 2.158s : parent [ main] PG=X 0 (of 10) nodes running:
[junit4] 1> 2.171s : parent [ restarter] PORT: 32834
[junit4] 1> 2.224s 0.1s: N0 [main child 0] top: corrupt unknown files
[junit4] 1> 2.233s : parent [ restarter] MDW: corrupt unknown files
[junit4] 1> 2.413s 0.3s: idle P0 [main child 0] top: now init primary
[junit4] 1> 2.421s 0.3s: idle P0 [main child 0] IWC:
[junit4] 1> 2.422s : parent [ restarter] analyzer=org.apache.lucene.analysis.MockAnalyzer
[junit4] 1> 2.425s : parent [ restarter] ramBufferSizeMB=16.0
[junit4] 1> 2.425s : parent [ restarter] maxBufferedDocs=-1
[junit4] 1> 2.425s : parent [ restarter] mergedSegmentWarmer=org.apache.lucene.replicator.nrt.PreCopyMergedSegmentWarmer@7a1f25a6
[junit4] 1> 2.426s : parent [ restarter] delPolicy=org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy
[junit4] 1> 2.426s : parent [ restarter] commit=null
[junit4] 1> 2.426s : parent [ restarter] openMode=CREATE_OR_APPEND
[junit4] 1> 2.426s : parent [ restarter] similarity=org.apache.lucene.search.similarities.AssertingSimilarity
[junit4] 1> 2.426s : parent [ restarter] mergeScheduler=ConcurrentMergeScheduler: maxThreadCount=2, maxMergeCount=3, ioThrottle=true
[junit4] 1> 2.426s : parent [ restarter] codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=39, maxMBSortInHeap=6.153388661207469
[junit4] 1> 2.427s : parent [ restarter] infoStream=org.apache.lucene.util.InfoStream$NoOutput
[junit4] 1> 2.427s : parent [ restarter] mergePolicy=[LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=3, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 1> 2.427s : parent [ restarter] indexerThreadPool=org.apache.lucene.index.DocumentsWriterPerThreadPool@55e7beb7
[junit4] 1> 2.427s : parent [ restarter] readerPooling=true
[junit4] 1> 2.427s : parent [ restarter] perThreadHardLimitMB=1945
[junit4] 1> 2.427s : parent [ restarter] useCompoundFile=true
[junit4] 1> 2.427s : parent [ restarter] commitOnClose=true
[junit4] 1> 2.428s : parent [ restarter] indexSort=null
[junit4] 1> 2.428s : parent [ restarter] checkPendingFlushOnUpdate=true
[junit4] 1> 2.428s : parent [ restarter] softDeletesField=null
[junit4] 1> 2.428s : parent [ restarter] readerAttributes={}
[junit4] 1> 2.428s : parent [ restarter] writer=org.apache.lucene.index.IndexWriter@29de0dd6
[junit4] 1> 2.441s : parent [ restarter]
[junit4] 1> 2.442s 0.3s: idle P0 [main child 0] dir:
[junit4] 1> 2.447s : parent [ restarter] MockDirectoryWrapper(NIOFSDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index0-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@31ee3c4e)
[junit4] 1> 2.448s 0.3s: idle P0 [main child 0] commitData: []
[junit4] 1> 2.449s 0.3s: idle P0 [main child 0] add initial commitData version=0
[junit4] 1> 2.451s 0.3s: idle P0 [main child 0] now forcePrimaryVersion to version=1
[junit4] 1> 2.477s 0.3s: idle P0 [main child 0] top: switch to infos=segments: version=2
[junit4] 1> 2.494s 0.3s: idle P0 [main child 0] top: set copyState primaryGen=0 version=2 files=[]
[junit4] 1> 2.495s 0.3s: idle P0 [main child 0] init: infos version=2
[junit4] 1> 2.508s : parent [ restarter]
[junit4] 1> 2.509s : parent [ restarter] COMMIT VERSION: 0
[junit4] 1> 2.509s : parent [ restarter]
[junit4] 1> 2.510s : parent [ restarter] INFOS VERSION: 2
[junit4] 1> 2.510s 0.4s: idle P0 [main child 0] top: will crash after 40.998 seconds
[junit4] 1> 2.524s : parent [ restarter]
[junit4] 1> 2.524s : parent [ restarter] NODE STARTED
[junit4] 1> 2.535s : parent [ restarter] top: node=0 started at tcpPort=32834 initCommitVersion=0 initInfosVersion=2
[junit4] 1> 2.536s : parent [ restarter] top: record transLogLoc: version=2 loc=0
[junit4] 1> 2.536s : parent [ restarter] top: record marker count: version=2 count=0
[junit4] 1> 2.537s : parent [ restarter] top: now change lastPrimaryVersion from 0 to 2; startup marker count 0
[junit4] 1> 2.537s : parent [ restarter] top: start translog replay 0 (version=0) to 0 (translog end)
[junit4] 1> 2.588s : parent [ restarter] top: done translog replay; took 51.620597 msec; now publish primary
[junit4] 1> 2.589s : parent [ restarter] top: send 0 replicas to primary
[junit4] 1> 2.588s 0.4s: idle P0 [ indexing] start handling indexing socket=Socket[addr=/127.0.0.1,port=57106,localport=32834]
[junit4] 1> 2.590s : parent [ main] PG=0 1 (of 10) nodes running: P0
[junit4] 1> 2.590s : parent [ main] top: now commit node=P0 tcpPort=32834
[junit4] 1> 2.632s 0.5s: idle P0 [ set repls] top: set replicasIDs=[] tcpPorts=[]
[junit4] 1> 2.633s : parent [Searcher node=P0 tcpPort=32834] top: searcher: record search hitCount version=2 hitCount=0 node=P0 tcpPort=32834
[junit4] 1> 2.633s 0.5s: idle P0 [ commit] top: commit commitData={__version=2, __primaryGen=0}
[junit4] 1> 2.636s 0.5s: idle P0 [ indexing] start handling indexing socket=Socket[addr=/127.0.0.1,port=57114,localport=32834]
[junit4] 1> 2.644s 0.5s: idle P0 [ indexing] start handling indexing socket=Socket[addr=/127.0.0.1,port=57116,localport=32834]
[junit4] 1> 2.967s : parent [ restarter] node status p0(2.6s) x1(3.0s) x2(3.0s) x3(3.0s) x4(3.0s) x5(3.0s) x6(3.0s) x7(3.0s) x8(3.0s) x9(3.0s)
[junit4] 1> 2.967s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 2.967s : parent [ restarter] N1: top: start up: launch thread
[junit4] 1> 2.997s : parent [ start R1] N1: top: start up thread
[junit4] 1> 3.243s : parent [ start R1] JUnit version 4.12
[junit4] 1> 3.295s : parent [ restarter] node status p0(2.9s) s1(0.3s) x2(3.3s) x3(3.3s) x4(3.3s) x5(3.3s) x6(3.3s) x7(3.3s) x8(3.3s) x9(3.3s)
[junit4] 1> 3.295s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 3.295s : parent [ restarter] N6: top: start up: launch thread
[junit4] 1> 3.306s : parent [ start R6] N6: top: start up thread
[junit4] 1> Indexer: now pause for 1826 msec...
[junit4] 1> 3.508s : parent [ restarter] node status p0(3.1s) s1(0.5s) x2(3.5s) x3(3.5s) x4(3.5s) x5(3.5s) s6(0.2s) x7(3.5s) x8(3.5s) x9(3.5s)
[junit4] 1> 3.508s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 3.509s : parent [ restarter] N9: top: start up: launch thread
[junit4] 1> Indexer: now pause for 1826 msec...
[junit4] 1> 3.533s : parent [ start R9] N9: top: start up thread
[junit4] 1> 3.620s : parent [ restarter] node status p0(3.2s) s1(0.6s) x2(3.6s) x3(3.6s) x4(3.6s) x5(3.6s) s6(0.3s) x7(3.6s) x8(3.6s) s9(0.1s)
[junit4] 1> 3.621s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 3.621s : parent [ restarter] N7: top: start up: launch thread
[junit4] 1> 3.631s : parent [ start R7] N7: top: start up thread
[junit4] 1> 3.681s : parent [ start R6] JUnit version 4.12
[junit4] 1> 3.791s : parent [ start R9] JUnit version 4.12
[junit4] 1> 3.998s : parent [ start R7] JUnit version 4.12
[junit4] 1> 4.086s : parent [ restarter] node status p0(3.7s) s1(1.1s) x2(4.1s) x3(4.1s) x4(4.1s) x5(4.1s) s6(0.8s) s7(0.5s) x8(4.1s) s9(0.6s)
[junit4] 1> 4.086s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 4.086s : parent [ restarter] node 1 still starting
[junit4] 1> 4.269s : parent [ main] PG=0 1 (of 10) nodes running: P0
[junit4] 1> 4.322s : parent [ restarter] node status p0(3.9s) s1(1.3s) x2(4.3s) x3(4.3s) x4(4.3s) x5(4.3s) s6(1.0s) s7(0.7s) x8(4.3s) s9(0.8s)
[junit4] 1> 4.322s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 4.322s : parent [ restarter] N2: top: start up: launch thread
[junit4] 1> 4.332s : parent [ start R2] N2: top: start up thread
[junit4] 1> 4.399s : parent [ restarter] node status p0(4.0s) s1(1.4s) s2(0.1s) x3(4.4s) x4(4.4s) x5(4.4s) s6(1.1s) s7(0.8s) x8(4.4s) s9(0.9s)
[junit4] 1> 4.399s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 4.399s : parent [ restarter] N4: top: start up: launch thread
[junit4] 1> 4.429s : parent [ start R4] N4: top: start up thread
[junit4] 1> 4.601s : parent [ restarter] node status p0(4.2s) s1(1.6s) s2(0.3s) x3(4.6s) s4(0.2s) x5(4.6s) s6(1.3s) s7(1.0s) x8(4.6s) s9(1.1s)
[junit4] 1> 4.601s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 4.601s : parent [ restarter] N5: top: start up: launch thread
[junit4] 1> 4.623s : parent [ start R5] N5: top: start up thread
[junit4] 1> 4.748s : parent [ restarter] node status p0(4.3s) s1(1.7s) s2(0.4s) x3(4.7s) s4(0.3s) s5(0.1s) s6(1.4s) s7(1.1s) x8(4.7s) s9(1.2s)
[junit4] 1> 4.748s : parent [ restarter] downNodes=[1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 4.748s : parent [ restarter] N8: top: start up: launch thread
[junit4] 1> 4.755s : parent [ start R8] N8: top: start up thread
[junit4] 1> 4.782s : parent [ start R5] JUnit version 4.12
[junit4] 1> 4.811s : parent [ restarter] top: full cluster crash
[junit4] 1> 4.812s : parent [ restarter] top: N0: top: now crash node
[junit4] 1> 4.852s : parent [ start R2] JUnit version 4.12
[junit4] 1> 5.057s : parent [ start R4] JUnit version 4.12
[junit4] 1> 5.182s : parent [ pump0] now wait for process java.lang.UNIXProcess@54064533
[junit4] 1> 5.186s : parent [ pump0] done wait for process java.lang.UNIXProcess@54064533
[junit4] 1> 5.186s : parent [ pump0] exit value=143 willCrash=true
[junit4] 1> 5.187s : parent [ pump0] top: P0 tcpPort=32834: primary process finished
[junit4] 1> 5.232s : parent [ restarter] top: N0: top: done crash node
[junit4] 1> 5.233s : parent [ restarter] N1: top: wait for startup so we can crash...
[junit4] 1> 5.257s : parent [ start R1] .
[junit4] 1> 5.258s : parent [ start R1] PORT: 37103
[junit4] 1> Indexer: done pause for a bit...
[junit4] 1> 5.272s : parent [ indexer p0] top: indexer lost connection to primary
[junit4] 1> 5.324s : parent [ start R8] JUnit version 4.12
[junit4] 1> Indexer: done pause for a bit...
[junit4] 1> 5.356s : parent [ indexer p0] top: indexer lost connection to primary
[junit4] 1> 5.392s 0.1s: N1 [main child 1] top: corrupt unknown files
[junit4] 1> 5.396s : parent [ start R1] MDW: corrupt unknown files
[junit4] 1> 5.424s 0.2s: idle R1 [main child 1] top: init replica dir=MockDirectoryWrapper(MMapDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index1-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@58f4b779)
[junit4] 1> 5.436s 0.2s: init R1 [main child 1] top: will rate limit file fetch to 5.42 MB/sec
[junit4] 1> 5.436s 0.2s: init R1 [main child 1] top: now start
[junit4] 1> 5.438s 0.2s: init R1 [main child 1] top: init: no segments in index
[junit4] 1> 5.452s 0.2s: init R1 [main child 1] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 5.461s 0.2s: init R1 [main child 1] now delete 0 files: []
[junit4] 1> 5.464s 0.2s: init R1 [main child 1] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 5.464s 0.2s: init R1 [main child 1] top: myPrimaryGen=-1
[junit4] 1> 5.470s 0.2s: init R1 [main child 1] top: same primary as before
[junit4] 1> 5.471s 0.2s: init R1 [main child 1] send new_replica to primary tcpPort=32834
[junit4] 1> 5.488s 0.2s: init R1 [main child 1] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 5.489s 0.2s: init R1 [main child 1] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 5.585s 0.3s: init R1 [main child 1] top: done start
[junit4] 1> 5.596s : parent [ main] PG=X 0 (of 10) nodes running:
[junit4] 1> 5.604s : parent [ start R1]
[junit4] 1> 5.604s : parent [ start R1] INFOS VERSION: 0
[junit4] 1> 5.604s 0.4s: idle R1 [main child 1] top: will crash after 18.191 seconds
[junit4] 1> 5.627s : parent [ start R1]
[junit4] 1> 5.627s : parent [ start R1] NODE STARTED
[junit4] 1> 5.655s : parent [ start R1] top: node=1 started at tcpPort=37103 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 5.693s : parent [ restarter] N1: top: done wait for startup
[junit4] 1> 5.693s : parent [ restarter] top: N1: top: now crash node
[junit4] 1> 5.944s : parent [ main] PG=X 1 (of 10) nodes running: R1
[junit4] 1> 6.101s : parent [ pump1] now wait for process java.lang.UNIXProcess@35ff0009
[junit4] 1> 6.102s : parent [ pump1] done wait for process java.lang.UNIXProcess@35ff0009
[junit4] 1> 6.102s : parent [ pump1] exit value=143 willCrash=true
[junit4] 1> 6.103s : parent [ pump1] top: R1 tcpPort=37103: replica process finished
[junit4] 1> 6.124s : parent [ restarter] top: N1: top: done crash node
[junit4] 1> 6.124s : parent [ restarter] N2: top: wait for startup so we can crash...
[junit4] 1> 6.420s : parent [ start R6] .
[junit4] 1> 6.420s : parent [ start R6] PORT: 45805
[junit4] 1> 6.444s : parent [ main] PG=X 0 (of 10) nodes running:
[junit4] 1> 6.560s 0.2s: N6 [main child 6] top: corrupt unknown files
[junit4] 1> 6.576s : parent [ start R6] MDW: corrupt unknown files
[junit4] 1> 6.581s 0.2s: idle R6 [main child 6] top: init replica dir=MockDirectoryWrapper(MMapDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index6-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7b556f99)
[junit4] 1> 6.608s 0.2s: init R6 [main child 6] top: will rate limit file fetch to 9.92 MB/sec
[junit4] 1> 6.609s 0.2s: init R6 [main child 6] top: now start
[junit4] 1> 6.632s 0.2s: init R6 [main child 6] top: init: no segments in index
[junit4] 1> 6.634s 0.2s: init R6 [main child 6] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 6.636s 0.2s: init R6 [main child 6] now delete 0 files: []
[junit4] 1> 6.644s 0.2s: init R6 [main child 6] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 6.644s 0.2s: init R6 [main child 6] top: myPrimaryGen=-1
[junit4] 1> 6.645s 0.2s: init R6 [main child 6] top: same primary as before
[junit4] 1> 6.649s 0.3s: init R6 [main child 6] send new_replica to primary tcpPort=32834
[junit4] 1> 6.655s 0.3s: init R6 [main child 6] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 6.666s 0.3s: init R6 [main child 6] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 6.770s 0.4s: init R6 [main child 6] top: done start
[junit4] 1> 6.786s : parent [ start R6]
[junit4] 1> 6.787s : parent [ start R6] INFOS VERSION: 0
[junit4] 1> 6.788s 0.4s: idle R6 [main child 6] top: will close after 25.982 seconds
[junit4] 1> 6.798s : parent [ start R6]
[junit4] 1> 6.800s : parent [ start R6] NODE STARTED
[junit4] 1> 6.800s : parent [ start R6] top: node=6 started at tcpPort=45805 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 6.849s : parent [Searcher node=R6 tcpPort=45805] top: searcher: record search hitCount version=0 hitCount=0 node=R6 tcpPort=45805
[junit4] 1> 7.166s : parent [ start R7] .
[junit4] 1> 7.177s : parent [ start R7] PORT: 46221
[junit4] 1> 7.193s : parent [ start R9] .
[junit4] 1> 7.193s : parent [ start R9] PORT: 37069
[junit4] 1> 7.238s 0.1s: N7 [main child 7] top: corrupt unknown files
[junit4] 1> 7.242s : parent [ start R7] MDW: corrupt unknown files
[junit4] 1> 7.252s 0.1s: idle R7 [main child 7] top: init replica dir=MockDirectoryWrapper(SimpleFSDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index7-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5970f72d)
[junit4] 1> 7.244s 0.1s: N9 [main child 9] top: corrupt unknown files
[junit4] 1> 7.262s : parent [ start R9] MDW: corrupt unknown files
[junit4] 1> 7.277s 0.1s: idle R9 [main child 9] top: init replica dir=MockDirectoryWrapper(SimpleFSDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index9-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5970f72d)
[junit4] 1> 7.279s 0.1s: init R7 [main child 7] top: will rate limit file fetch to 15.24 MB/sec
[junit4] 1> 7.280s 0.1s: init R7 [main child 7] top: now start
[junit4] 1> 7.305s 0.1s: init R9 [main child 9] top: will rate limit file fetch to 19.85 MB/sec
[junit4] 1> 7.305s 0.1s: init R9 [main child 9] top: now start
[junit4] 1> 7.308s 0.1s: init R7 [main child 7] top: init: no segments in index
[junit4] 1> 7.309s 0.1s: init R7 [main child 7] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.311s 0.1s: init R7 [main child 7] now delete 0 files: []
[junit4] 1> 7.312s 0.2s: init R7 [main child 7] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.313s 0.2s: init R7 [main child 7] top: myPrimaryGen=-1
[junit4] 1> 7.314s 0.2s: init R7 [main child 7] top: same primary as before
[junit4] 1> 7.314s 0.2s: init R7 [main child 7] send new_replica to primary tcpPort=32834
[junit4] 1> 7.325s 0.2s: init R7 [main child 7] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 7.333s 0.2s: init R9 [main child 9] top: init: no segments in index
[junit4] 1> 7.333s 0.2s: init R7 [main child 7] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 7.336s 0.2s: init R9 [main child 9] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.352s 0.2s: init R9 [main child 9] now delete 0 files: []
[junit4] 1> 7.353s 0.2s: init R9 [main child 9] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.354s 0.2s: init R9 [main child 9] top: myPrimaryGen=-1
[junit4] 1> 7.355s 0.2s: init R9 [main child 9] top: same primary as before
[junit4] 1> 7.355s 0.2s: init R9 [main child 9] send new_replica to primary tcpPort=32834
[junit4] 1> 7.364s 0.2s: init R9 [main child 9] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 7.365s 0.2s: init R9 [main child 9] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 7.425s 0.3s: init R7 [main child 7] top: done start
[junit4] 1> 7.436s : parent [ start R7]
[junit4] 1> 7.436s : parent [ start R7] INFOS VERSION: 0
[junit4] 1> 7.436s 0.3s: idle R7 [main child 7] top: will close after 45.189 seconds
[junit4] 1> 7.419s 0.2s: init R9 [main child 9] top: done start
[junit4] 1> 7.450s : parent [ start R7]
[junit4] 1> 7.450s : parent [ start R7] NODE STARTED
[junit4] 1> 7.452s : parent [ start R9]
[junit4] 1> 7.452s : parent [ start R9] INFOS VERSION: 0
[junit4] 1> 7.452s 0.3s: idle R9 [main child 9] top: will crash after 32.865 seconds
[junit4] 1> 7.460s : parent [ start R9]
[junit4] 1> 7.460s : parent [ start R9] NODE STARTED
[junit4] 1> 7.460s : parent [ start R7] top: node=7 started at tcpPort=46221 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 7.464s : parent [ start R9] top: node=9 started at tcpPort=37069 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 7.622s : parent [ start R2] .
[junit4] 1> 7.622s : parent [ start R2] PORT: 36326
[junit4] 1> 7.671s : parent [ start R5] .
[junit4] 1> 7.677s : parent [ start R5] PORT: 35263
[junit4] 1> 7.682s 0.1s: N2 [main child 2] top: corrupt unknown files
[junit4] 1> 7.690s : parent [ start R2] MDW: corrupt unknown files
[junit4] 1> 7.693s 0.1s: idle R2 [main child 2] top: init replica dir=MockDirectoryWrapper(MMapDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index2-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1934e9c8)
[junit4] 1> 7.708s 0.1s: init R2 [main child 2] top: will rate limit file fetch to 11.44 MB/sec
[junit4] 1> 7.708s 0.1s: init R2 [main child 2] top: now start
[junit4] 1> 7.718s 0.1s: init R2 [main child 2] top: init: no segments in index
[junit4] 1> 7.719s 0.1s: init R2 [main child 2] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.720s 0.1s: init R2 [main child 2] now delete 0 files: []
[junit4] 1> 7.722s 0.1s: init R2 [main child 2] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.722s 0.1s: init R2 [main child 2] top: myPrimaryGen=-1
[junit4] 1> 7.723s 0.1s: init R2 [main child 2] top: same primary as before
[junit4] 1> 7.723s 0.1s: init R2 [main child 2] send new_replica to primary tcpPort=32834
[junit4] 1> 7.727s 0.1s: init R2 [main child 2] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 7.728s 0.1s: init R2 [main child 2] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 7.774s : parent [ main] PG=X 3 (of 10) nodes running: R6 R7 R9
[junit4] 1> 7.775s 0.2s: init R2 [main child 2] top: done start
[junit4] 1> 7.784s : parent [ start R2]
[junit4] 1> 7.784s : parent [ start R2] INFOS VERSION: 0
[junit4] 1> 7.784s 0.2s: idle R2 [main child 2] top: will close after 56.721 seconds
[junit4] 1> 7.789s : parent [ start R2]
[junit4] 1> 7.790s : parent [ start R2] NODE STARTED
[junit4] 1> 7.791s : parent [ start R2] top: node=2 started at tcpPort=36326 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 7.782s 0.1s: N5 [main child 5] top: corrupt unknown files
[junit4] 1> 7.794s : parent [ start R5] MDW: corrupt unknown files
[junit4] 1> 7.795s : parent [ restarter] N2: top: done wait for startup
[junit4] 1> 7.795s : parent [ restarter] top: N2: top: now crash node
[junit4] 1> 7.806s 0.1s: idle R5 [main child 5] top: init replica dir=MockDirectoryWrapper(SimpleFSDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index5-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5970f72d)
[junit4] 1> 7.822s 0.2s: init R5 [main child 5] top: will rate limit file fetch to 8.87 MB/sec
[junit4] 1> 7.824s 0.2s: init R5 [main child 5] top: now start
[junit4] 1> 7.854s 0.2s: init R5 [main child 5] top: init: no segments in index
[junit4] 1> 7.857s 0.2s: init R5 [main child 5] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.858s 0.2s: init R5 [main child 5] now delete 0 files: []
[junit4] 1> 7.862s 0.2s: init R5 [main child 5] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.862s 0.2s: init R5 [main child 5] top: myPrimaryGen=-1
[junit4] 1> 7.862s 0.2s: init R5 [main child 5] top: same primary as before
[junit4] 1> 7.863s 0.2s: init R5 [main child 5] send new_replica to primary tcpPort=32834
[junit4] 1> 7.866s 0.2s: init R5 [main child 5] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 7.867s 0.2s: init R5 [main child 5] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 7.869s : parent [ start R8] .
[junit4] 1> 7.869s : parent [ start R8] PORT: 32905
[junit4] 1> 7.908s : parent [ start R4] .
[junit4] 1> 7.909s : parent [ start R4] PORT: 46299
[junit4] 1> 7.915s 0.2s: init R5 [main child 5] top: done start
[junit4] 1> 7.943s 0.1s: N8 [main child 8] top: corrupt unknown files
[junit4] 1> 7.949s : parent [ start R8] MDW: corrupt unknown files
[junit4] 1> 7.956s 0.1s: idle R8 [main child 8] top: init replica dir=MockDirectoryWrapper(MMapDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index8-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@75ae4b61)
[junit4] 1> 7.960s : parent [ start R5]
[junit4] 1> 7.960s : parent [ start R5] INFOS VERSION: 0
[junit4] 1> 7.960s 0.3s: idle R5 [main child 5] top: will crash after 56.741 seconds
[junit4] 1> 7.960s 0.1s: N4 [main child 4] top: corrupt unknown files
[junit4] 1> 7.965s : parent [ start R4] MDW: corrupt unknown files
[junit4] 1> 7.974s : parent [ start R5]
[junit4] 1> 7.974s : parent [ start R5] NODE STARTED
[junit4] 1> 7.975s : parent [ start R5] top: node=5 started at tcpPort=35263 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 7.977s 0.1s: init R8 [main child 8] top: will rate limit file fetch to 17.82 MB/sec
[junit4] 1> 7.977s 0.1s: init R8 [main child 8] top: now start
[junit4] 1> 7.980s 0.1s: idle R4 [main child 4] top: init replica dir=MockDirectoryWrapper(SimpleFSDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/lucene/build/replicator/test/J2/temp/lucene.replicator.nrt.TestStressNRTReplication_AB64A8554B0B5F2A-001/index4-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5970f72d)
[junit4] 1> 7.979s 0.1s: init R8 [main child 8] top: init: no segments in index
[junit4] 1> 7.981s 0.1s: init R8 [main child 8] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.982s 0.1s: init R8 [main child 8] now delete 0 files: []
[junit4] 1> 7.987s 0.1s: init R8 [main child 8] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 7.988s 0.1s: init R8 [main child 8] top: myPrimaryGen=-1
[junit4] 1> 7.988s 0.1s: init R8 [main child 8] top: same primary as before
[junit4] 1> 7.989s 0.1s: init R8 [main child 8] send new_replica to primary tcpPort=32834
[junit4] 1> 7.993s 0.1s: init R8 [main child 8] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 7.994s 0.1s: init R4 [main child 4] top: will rate limit file fetch to 9.69 MB/sec
[junit4] 1> 7.994s 0.1s: init R4 [main child 4] top: now start
[junit4] 1> 7.994s 0.1s: init R8 [main child 8] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 8.000s 0.1s: init R4 [main child 4] top: init: no segments in index
[junit4] 1> 8.001s 0.1s: init R4 [main child 4] top: delete unknown files on init: all files=[write.lock]
[junit4] 1> 8.002s 0.1s: init R4 [main child 4] now delete 0 files: []
[junit4] 1> 8.004s 0.1s: init R4 [main child 4] top: done delete unknown files on init: all files=[write.lock]
[junit4] 1> 8.004s 0.1s: init R4 [main child 4] top: myPrimaryGen=-1
[junit4] 1> 8.004s 0.1s: init R4 [main child 4] top: same primary as before
[junit4] 1> 8.005s 0.1s: init R4 [main child 4] send new_replica to primary tcpPort=32834
[junit4] 1> 8.012s 0.1s: init R4 [main child 4] ignoring exc java.net.ConnectException: Connection refused (Connection refused) sending new_replica to primary tcpPort=32834
[junit4] 1> 8.013s 0.1s: init R4 [main child 4] SegmentInfosSearcherManager.init: use incoming infos=segments:
[junit4] 1> 8.032s 0.2s: init R8 [main child 8] top: done start
[junit4] 1> 8.034s : parent [ start R8]
[junit4] 1> 8.034s : parent [ start R8] INFOS VERSION: 0
[junit4] 1> 8.035s 0.2s: idle R8 [main child 8] top: will crash after 35.791 seconds
[junit4] 1> 8.036s : parent [ start R8]
[junit4] 1> 8.036s : parent [ start R8] NODE STARTED
[junit4] 1> 8.037s : parent [ start R8] top: node=8 started at tcpPort=32905 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 8.054s 0.1s: init R4 [main child 4] top: done start
[junit4] 1> 8.092s : parent [ start R4]
[junit4] 1> 8.092s : parent [ start R4] INFOS VERSION: 0
[junit4] 1> 8.092s 0.2s: idle R4 [main child 4] top: will crash after 51.743 seconds
[junit4] 1> 8.093s : parent [ start R4]
[junit4] 1> 8.093s : parent [ start R4] NODE STARTED
[junit4] 1> 8.095s : parent [ start R4] top: node=4 started at tcpPort=46299 initCommitVersion=-1 initInfosVersion=0
[junit4] 1> 8.120s : parent [ main] PG=X 7 (of 10) nodes running: R2 R4 R5 R6 R7 R8 R9
[junit4] 1> 8.155s : parent [ pump2] now wait for process java.lang.UNIXProcess@7d80c82
[junit4] 1> 8.158s : parent [ pump2] done wait for process java.lang.UNIXProcess@7d80c82
[junit4] 1> 8.158s : parent [ pump2] exit value=143 willCrash=false
[junit4] 1> 8.159s : parent [ pump2] top: R2 tcpPort=36326: replica process finished
[junit4] 1> 8.159s : parent [ restarter] top: N2: top: done crash node
[junit4] 1> 8.159s : parent [ restarter] top: N4: top: now crash node
[junit4] 1> 8.502s : parent [ pump4] now wait for process java.lang.UNIXProcess@5c539721
[junit4] 1> 8.503s : parent [ pump4] done wait for process java.lang.UNIXProcess@5c539721
[junit4] 1> 8.503s : parent [ pump4] exit value=143 willCrash=true
[junit4] 1> 8.504s : parent [ pump4] top: R4 tcpPort=46299: replica process finished
[junit4] 1> 8.504s : parent [ restarter] top: N4: top: done crash node
[junit4] 1> 8.504s : parent [ restarter] top: N5: top: now crash node
[junit4] 1> 8.564s : parent [ main] PG=X 5 (of 10) nodes running: R5 R6 R7 R8 R9
[junit4] 1> 8.879s : parent [ pump5] now wait for process java.lang.UNIXProcess@22d9f740
[junit4] 1> 8.881s : parent [ pump5] done wait for process java.lang.UNIXProcess@22d9f740
[junit4] 1> 8.881s : parent [ pump5] exit value=143 willCrash=true
[junit4] 1> 8.881s : parent [ pump5] top: R5 tcpPort=35263: replica process finished
[junit4] 1> 8.881s : parent [ restarter] top: N5: top: done crash node
[junit4] 1> 8.881s : parent [ restarter] top: N6: top: now crash node
[junit4] 1> 9.230s : parent [ pump6] now wait for process java.lang.UNIXProcess@16bd6fed
[junit4] 1> 9.232s : parent [ pump6] done wait for process java.lang.UNIXProcess@16bd6fed
[junit4] 1> 9.232s : parent [ pump6] exit value=143 willCrash=false
[junit4] 1> 9.232s : parent [ pump6] top: R6 tcpPort=45805: replica process finished
[junit4] 1> 9.232s : parent [ restarter] top: N6: top: done crash node
[junit4] 1> 9.233s : parent [ restarter] top: N7: top: now crash node
[junit4] 1> 9.445s : parent [ main] PG=X 3 (of 10) nodes running: R7 R8 R9
[junit4] 1> 9.568s : parent [ pump7] now wait for process java.lang.UNIXProcess@52fadad
[junit4] 1> 9.569s : parent [ pump7] done wait for process java.lang.UNIXProcess@52fadad
[junit4] 1> 9.569s : parent [ pump7] exit value=143 willCrash=false
[junit4] 1> 9.569s : parent [ pump7] top: R7 tcpPort=46221: replica process finished
[junit4] 1> 9.570s : parent [ restarter] top: N7: top: done crash node
[junit4] 1> 9.570s : parent [ restarter] top: N8: top: now crash node
[junit4] 1> 9.898s : parent [ pump8] now wait for process java.lang.UNIXProcess@36329d4b
[junit4] 1> 9.906s : parent [ pump8] done wait for process java.lang.UNIXProcess@36329d4b
[junit4] 1> 9.906s : parent [ pump8] exit value=143 willCrash=true
[junit4] 1> 9.906s : parent [ pump8] top: R8 tcpPort=32905: replica process finished
[junit4] 1> 9.906s : parent [ restarter] top: N8: top: done crash node
[junit4] 1> 9.906s : parent [ restarter] top: N9: top: now crash node
[junit4] 1> 10.245s : parent [ pump9] now wait for process java.lang.UNIXProcess@e945446
[junit4] 1> 10.253s : parent [ pump9] done wait for process java.lang.UNIXProcess@e945446
[junit4] 1> 10.253s : parent [ pump9] exit value=143 willCrash=true
[junit4] 1> 10.253s : parent [ pump9] top: R9 tcpPort=37069: replica process finished
[junit4] 1> 10.254s : parent [ restarter] top: N9: top: done crash node
[junit4] 1> 10.254s : parent [ restarter] node status x0(5.1s) x1(4.2s) x2(2.1s) x3(10.3s) x4(1.8s) x5(1.4s) x6(1.0s) x7(0.7s) x8(0.3s) x9(0.0s)
[junit4] 1> 10.254s : parent [ restarter] downNodes=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
[junit4] 1> 10.254s : parent [ restarter] top: primary crashed; now pick replica to promote
[junit4] 1> 10.254s : parent [ restarter] top: no replicas running; skipping primary promotion
[junit4] 1> 10.254s : parent [ restarter] N0: top: cold start as primary
[junit4] 1> 10.254s : parent [ restarter] 0: top: startPrimary lastPrimaryVersion=2
[junit4] 1> 10.483s : parent [ restarter] JUnit version 4.12
[junit4] 1> 10.903s : parent [ main] PG=X 0 (of 10) nodes running:
[junit4] 1> 11.695
[...truncated too long message...]
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 3) Thread[id=1568, name=Connector-Scheduler-6805c61e, state=WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 4) Thread[id=1471, name=qtp1815760705-1471, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=1468, name=qtp1815760705-1468, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=1466, name=qtp1815760705-1466, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=1470, name=qtp1815760705-1470, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 8) Thread[id=1469, name=qtp1815760705-1469, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 9) Thread[id=1464, name=qtp1815760705-1464, state=RUNNABLE, group=TGRP-ShardSplitTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$$Lambda$48/1580687262.run(Unknown Source)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 10) Thread[id=1467, name=qtp1815760705-1467, state=TIMED_WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([97B926D1D108389F]:0)
[junit4] Completed [25/889 (1!)] on J1 in 914.68s, 11 tests, 3 errors <<< FAILURES!
[...truncated 12197 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/build.xml:662: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-8.x/checkout/build.xml:586: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid22088.hprof
Total time: 438 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any