You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/08/22 09:30:26 UTC
[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 127 -
Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/127/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC
5 tests failed.
FAILED: org.apache.solr.cloud.TestLocalFSCloudBackupRestore.test
Error Message:
Error from server at http://127.0.0.1:59418/solr: ADDREPLICA failed to create replica
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59418/solr: ADDREPLICA failed to create replica
at __randomizedtesting.SeedInfo.seed([1AD9EA1D24B4972A:928DD5C78A48FAD2]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:275)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestReplicaProperties.test
Error Message:
Stack Trace:
java.lang.NullPointerException
at __randomizedtesting.SeedInfo.seed([1AD9EA1D24B4972A:928DD5C78A48FAD2]:0)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.getNodeName(JettySolrRunner.java:345)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:416)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestReplicaProperties
Error Message:
67 threads leaked from SUITE scope at org.apache.solr.cloud.TestReplicaProperties: 1) Thread[id=6808, name=TEST-TestReplicaProperties.test-seed#[1AD9EA1D24B4972A]-SendThread(127.0.0.1:59774), state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 2) Thread[id=6811, name=Thread-840, state=WAITING, group=TGRP-TestReplicaProperties] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1617) 3) Thread[id=6809, name=TEST-TestReplicaProperties.test-seed#[1AD9EA1D24B4972A]-EventThread, state=WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 4) Thread[id=6788, name=searcherExecutor-2022-thread-1, state=WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=6817, name=searcherExecutor-2033-thread-1, state=WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=6819, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=6798, name=qtp476268787-6798, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=6767, name=qtp1431813100-6767, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=6779, name=TEST-TestReplicaProperties.test-seed#[1AD9EA1D24B4972A]-EventThread, state=WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 10) Thread[id=6852, name=zkCallback-1355-thread-3, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=6821, name=qtp975157825-6821-acceptor-0@8998ace-ServerConnector@6212fb8c{HTTP/1.1,[http/1.1]}{127.0.0.1:59819}, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=6797, name=qtp476268787-6797, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=6804, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=6844, name=OverseerStateUpdate-98526288693166089-127.0.0.1:59796_-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:184) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=6782, name=Thread-831, state=WAITING, group=TGRP-TestReplicaProperties] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1617) 16) Thread[id=6793, name=qtp476268787-6793, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=6794, name=qtp476268787-6794, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=6823, name=qtp975157825-6823, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=6773, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=6837, name=TEST-TestReplicaProperties.test-seed#[1AD9EA1D24B4972A]-SendThread(127.0.0.1:59774), state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 21) Thread[id=6838, name=TEST-TestReplicaProperties.test-seed#[1AD9EA1D24B4972A]-EventThread, state=WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 22) Thread[id=6853, name=zkCallback-1361-thread-3, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=6765, name=qtp1431813100-6765, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=6780, name=zkCallback-1355-thread-1, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=6845, name=OverseerCollectionConfigSetProcessor-98526288693166089-127.0.0.1:59796_-n_0000000001, state=WAITING, group=Overseer collection creation process.] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210) at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357) at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=6790, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=6810, name=zkCallback-1361-thread-1, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=6822, name=qtp975157825-6822, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=6843, name=zkCallback-1355-thread-2, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=6799, name=org.eclipse.jetty.server.session.HashSessionManager@5cbe08beTimer, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=6854, name=zkCallback-1355-thread-4, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=6768, name=qtp1431813100-6768, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=6849, name=zkCallback-1367-thread-3, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=6851, name=zkCallback-1361-thread-2, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=6761, name=qtp1431813100-6761, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=6826, name=qtp975157825-6826, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 37) Thread[id=6855, name=zkCallback-1361-thread-4, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 38) Thread[id=6774, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=6820, name=qtp975157825-6820, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 40) Thread[id=6856, name=zkCallback-1355-thread-5, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 41) Thread[id=6778, name=TEST-TestReplicaProperties.test-seed#[1AD9EA1D24B4972A]-SendThread(127.0.0.1:59774), state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 42) Thread[id=6764, name=qtp1431813100-6764, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 43) Thread[id=6815, name=watches-1362-thread-1, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 44) Thread[id=6850, name=zkCallback-1367-thread-4, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 45) Thread[id=6786, name=watches-1356-thread-1, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 46) Thread[id=6848, name=zkCallback-1367-thread-2, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 47) Thread[id=6762, name=qtp1431813100-6762-acceptor-0@41c79a70-ServerConnector@59fff0b4{HTTP/1.1,[http/1.1]}{127.0.0.1:59796}, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 48) Thread[id=6783, name=coreContainerWorkExecutor-2017-thread-1, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 49) Thread[id=6803, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 50) Thread[id=6795, name=qtp476268787-6795, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 51) Thread[id=6839, name=zkCallback-1367-thread-1, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 52) Thread[id=6825, name=qtp975157825-6825, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 53) Thread[id=6792, name=qtp476268787-6792-acceptor-0@70b0c0a7-ServerConnector@3de8c677{HTTP/1.1,[http/1.1]}{127.0.0.1:59807}, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 54) Thread[id=6812, name=coreContainerWorkExecutor-2028-thread-1, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 55) Thread[id=6769, name=org.eclipse.jetty.server.session.HashSessionManager@6a768f10Timer, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 56) Thread[id=6791, name=qtp476268787-6791, state=RUNNABLE, group=TGRP-TestReplicaProperties] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 57) Thread[id=6833, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 58) Thread[id=6846, name=OverseerHdfsCoreFailoverThread-98526288693166089-127.0.0.1:59796_-n_0000000001, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139) at java.lang.Thread.run(Thread.java:748) 59) Thread[id=6763, name=qtp1431813100-6763, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 60) Thread[id=6832, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 61) Thread[id=6813, name=Scheduler-1979979029, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 62) Thread[id=6827, name=qtp975157825-6827, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 63) Thread[id=6828, name=org.eclipse.jetty.server.session.HashSessionManager@536fd3faTimer, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 64) Thread[id=6796, name=qtp476268787-6796, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 65) Thread[id=6824, name=qtp975157825-6824, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 66) Thread[id=6766, name=qtp1431813100-6766, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 67) Thread[id=6785, name=Scheduler-1221711587, state=TIMED_WAITING, group=TGRP-TestReplicaProperties] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 67 threads leaked from SUITE scope at org.apache.solr.cloud.TestReplicaProperties:
1) Thread[id=6808, name=TEST-TestReplicaProperties.test-seed#[1AD9EA1D24B4972A]-SendThread(127.0.0.1:59774), state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
2) Thread[id=6811, name=Thread-840, state=WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1617)
3) Thread[id=6809, name=TEST-TestReplicaProperties.test-seed#[1AD9EA1D24B4972A]-EventThread, state=WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
4) Thread[id=6788, name=searcherExecutor-2022-thread-1, state=WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=6817, name=searcherExecutor-2033-thread-1, state=WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=6819, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=6798, name=qtp476268787-6798, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=6767, name=qtp1431813100-6767, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=6779, name=TEST-TestReplicaProperties.test-seed#[1AD9EA1D24B4972A]-EventThread, state=WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
10) Thread[id=6852, name=zkCallback-1355-thread-3, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=6821, name=qtp975157825-6821-acceptor-0@8998ace-ServerConnector@6212fb8c{HTTP/1.1,[http/1.1]}{127.0.0.1:59819}, state=RUNNABLE, group=TGRP-TestReplicaProperties]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=6797, name=qtp476268787-6797, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=6804, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=6844, name=OverseerStateUpdate-98526288693166089-127.0.0.1:59796_-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)
at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:184)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=6782, name=Thread-831, state=WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1617)
16) Thread[id=6793, name=qtp476268787-6793, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
17) Thread[id=6794, name=qtp476268787-6794, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
18) Thread[id=6823, name=qtp975157825-6823, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
19) Thread[id=6773, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
20) Thread[id=6837, name=TEST-TestReplicaProperties.test-seed#[1AD9EA1D24B4972A]-SendThread(127.0.0.1:59774), state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
21) Thread[id=6838, name=TEST-TestReplicaProperties.test-seed#[1AD9EA1D24B4972A]-EventThread, state=WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
22) Thread[id=6853, name=zkCallback-1361-thread-3, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
23) Thread[id=6765, name=qtp1431813100-6765, state=RUNNABLE, group=TGRP-TestReplicaProperties]
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
24) Thread[id=6780, name=zkCallback-1355-thread-1, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
25) Thread[id=6845, name=OverseerCollectionConfigSetProcessor-98526288693166089-127.0.0.1:59796_-n_0000000001, state=WAITING, group=Overseer collection creation process.]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)
at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
at java.lang.Thread.run(Thread.java:748)
26) Thread[id=6790, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
27) Thread[id=6810, name=zkCallback-1361-thread-1, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
28) Thread[id=6822, name=qtp975157825-6822, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
29) Thread[id=6843, name=zkCallback-1355-thread-2, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
30) Thread[id=6799, name=org.eclipse.jetty.server.session.HashSessionManager@5cbe08beTimer, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
31) Thread[id=6854, name=zkCallback-1355-thread-4, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
32) Thread[id=6768, name=qtp1431813100-6768, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
33) Thread[id=6849, name=zkCallback-1367-thread-3, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
34) Thread[id=6851, name=zkCallback-1361-thread-2, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
35) Thread[id=6761, name=qtp1431813100-6761, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
36) Thread[id=6826, name=qtp975157825-6826, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
37) Thread[id=6855, name=zkCallback-1361-thread-4, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
38) Thread[id=6774, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
39) Thread[id=6820, name=qtp975157825-6820, state=RUNNABLE, group=TGRP-TestReplicaProperties]
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
40) Thread[id=6856, name=zkCallback-1355-thread-5, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
41) Thread[id=6778, name=TEST-TestReplicaProperties.test-seed#[1AD9EA1D24B4972A]-SendThread(127.0.0.1:59774), state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
42) Thread[id=6764, name=qtp1431813100-6764, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
43) Thread[id=6815, name=watches-1362-thread-1, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
44) Thread[id=6850, name=zkCallback-1367-thread-4, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
45) Thread[id=6786, name=watches-1356-thread-1, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
a
[...truncated too long message...]
sume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 24) Thread[id=6813, name=Scheduler-1979979029, state=WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 25) Thread[id=6828, name=org.eclipse.jetty.server.session.HashSessionManager@536fd3faTimer, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 26) Thread[id=6810, name=zkCallback-1361-thread-1, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 27) Thread[id=6843, name=zkCallback-1355-thread-2, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 28) Thread[id=6799, name=org.eclipse.jetty.server.session.HashSessionManager@5cbe08beTimer, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 29) Thread[id=6854, name=zkCallback-1355-thread-4, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 30) Thread[id=6849, name=zkCallback-1367-thread-3, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 31) Thread[id=6851, name=zkCallback-1361-thread-2, state=TIMED_WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 32) Thread[id=6785, name=Scheduler-1221711587, state=WAITING, group=TGRP-TestReplicaProperties]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([1AD9EA1D24B4972A]:0)
[junit4] Completed [140/730 (3!)] on J1 in 33.34s, 1 test, 3 errors <<< FAILURES!
[...truncated 45263 lines...]
[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 128 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/128/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC
2 tests failed.
FAILED: org.apache.solr.cloud.CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster
Error Message:
Document mismatch on target after sync expected:<2000> but was:<1901>
Stack Trace:
java.lang.AssertionError: Document mismatch on target after sync expected:<2000> but was:<1901>
at __randomizedtesting.SeedInfo.seed([2255F3A51CB77A0E:F610B8FCFBE1C9F5]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.apache.solr.cloud.CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster(CdcrBootstrapTest.java:309)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter
Error Message:
Stack Trace:
java.lang.AssertionError
at __randomizedtesting.SeedInfo.seed([2255F3A51CB77A0E:7DB1DE9277BBE94B]:0)
at org.junit.Assert.fail(Assert.java:92)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertTrue(Assert.java:54)
at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:74)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11201 lines...]
[junit4] Suite: org.apache.solr.cloud.CdcrBootstrapTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_2255F3A51CB77A0E-001/init-core-data-001
[junit4] 2> 388683 WARN (SUITE-CdcrBootstrapTest-seed#[2255F3A51CB77A0E]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=27 numCloses=27
[junit4] 2> 388683 INFO (SUITE-CdcrBootstrapTest-seed#[2255F3A51CB77A0E]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 388687 INFO (SUITE-CdcrBootstrapTest-seed#[2255F3A51CB77A0E]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 388687 INFO (SUITE-CdcrBootstrapTest-seed#[2255F3A51CB77A0E]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 388689 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[2255F3A51CB77A0E]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testBootstrapWithSourceCluster
[junit4] 2> 388689 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[2255F3A51CB77A0E]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_2255F3A51CB77A0E-001/cdcr-target-001
[junit4] 2> 388690 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[2255F3A51CB77A0E]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 388692 INFO (Thread-724) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 388692 INFO (Thread-724) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 388700 ERROR (Thread-724) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 388798 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[2255F3A51CB77A0E]) [ ] o.a.s.c.ZkTestServer start zk server on port:56165
[junit4] 2> 388839 INFO (jetty-launcher-562-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 388843 INFO (jetty-launcher-562-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6e1897e8{/solr,null,AVAILABLE}
[junit4] 2> 388845 INFO (jetty-launcher-562-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@782aa177{HTTP/1.1,[http/1.1]}{127.0.0.1:56167}
[junit4] 2> 388845 INFO (jetty-launcher-562-thread-1) [ ] o.e.j.s.Server Started @392205ms
[junit4] 2> 388845 INFO (jetty-launcher-562-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=56167}
[junit4] 2> 388845 ERROR (jetty-launcher-562-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 388846 INFO (jetty-launcher-562-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.1.0
[junit4] 2> 388846 INFO (jetty-launcher-562-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 388846 INFO (jetty-launcher-562-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 388846 INFO (jetty-launcher-562-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-22T18:57:38.395Z
[junit4] 2> 388866 INFO (jetty-launcher-562-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 388881 INFO (jetty-launcher-562-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56165/solr
[junit4] 2> 389130 INFO (jetty-launcher-562-thread-1) [n:127.0.0.1:56167_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 389131 INFO (jetty-launcher-562-thread-1) [n:127.0.0.1:56167_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56167_solr
[junit4] 2> 389133 INFO (jetty-launcher-562-thread-1) [n:127.0.0.1:56167_solr ] o.a.s.c.Overseer Overseer (id=98528674332344323-127.0.0.1:56167_solr-n_0000000000) starting
[junit4] 2> 389154 INFO (jetty-launcher-562-thread-1) [n:127.0.0.1:56167_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56167_solr
[junit4] 2> 389156 INFO (OverseerStateUpdate-98528674332344323-127.0.0.1:56167_solr-n_0000000000) [n:127.0.0.1:56167_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 389313 INFO (jetty-launcher-562-thread-1) [n:127.0.0.1:56167_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56167.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21fd5831
[junit4] 2> 389322 INFO (jetty-launcher-562-thread-1) [n:127.0.0.1:56167_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56167.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21fd5831
[junit4] 2> 389322 INFO (jetty-launcher-562-thread-1) [n:127.0.0.1:56167_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56167.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21fd5831
[junit4] 2> 389323 INFO (jetty-launcher-562-thread-1) [n:127.0.0.1:56167_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_2255F3A51CB77A0E-001/cdcr-target-001/node1/.
[junit4] 1> Target zkHost = 127.0.0.1:56165/solr
[junit4] 2> 389369 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[2255F3A51CB77A0E]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_2255F3A51CB77A0E-001/cdcr-source-001
[junit4] 2> 389369 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[2255F3A51CB77A0E]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 389370 INFO (Thread-734) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 389370 INFO (Thread-734) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 389376 ERROR (Thread-734) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 389476 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[2255F3A51CB77A0E]) [ ] o.a.s.c.ZkTestServer start zk server on port:56174
[junit4] 2> 389513 INFO (jetty-launcher-574-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 389515 INFO (jetty-launcher-574-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2dfa2a8{/solr,null,AVAILABLE}
[junit4] 2> 389515 INFO (jetty-launcher-574-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@27de6a23{HTTP/1.1,[http/1.1]}{127.0.0.1:56176}
[junit4] 2> 389515 INFO (jetty-launcher-574-thread-1) [ ] o.e.j.s.Server Started @392875ms
[junit4] 2> 389515 INFO (jetty-launcher-574-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=56176}
[junit4] 2> 389516 ERROR (jetty-launcher-574-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 389516 INFO (jetty-launcher-574-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.1.0
[junit4] 2> 389516 INFO (jetty-launcher-574-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 389516 INFO (jetty-launcher-574-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 389516 INFO (jetty-launcher-574-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-22T18:57:39.065Z
[junit4] 2> 389520 INFO (jetty-launcher-574-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 389532 INFO (jetty-launcher-574-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56174/solr
[junit4] 2> 389739 INFO (jetty-launcher-574-thread-1) [n:127.0.0.1:56176_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 389740 INFO (jetty-launcher-574-thread-1) [n:127.0.0.1:56176_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56176_solr
[junit4] 2> 389742 INFO (jetty-launcher-574-thread-1) [n:127.0.0.1:56176_solr ] o.a.s.c.Overseer Overseer (id=98528674376843267-127.0.0.1:56176_solr-n_0000000000) starting
[junit4] 2> 389759 INFO (jetty-launcher-574-thread-1) [n:127.0.0.1:56176_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56176_solr
[junit4] 2> 389765 INFO (zkCallback-581-thread-1-processing-n:127.0.0.1:56176_solr) [n:127.0.0.1:56176_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 389878 INFO (jetty-launcher-574-thread-1) [n:127.0.0.1:56176_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56176.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21fd5831
[junit4] 2> 389888 INFO (jetty-launcher-574-thread-1) [n:127.0.0.1:56176_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56176.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21fd5831
[junit4] 2> 389888 INFO (jetty-launcher-574-thread-1) [n:127.0.0.1:56176_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56176.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21fd5831
[junit4] 2> 389902 INFO (jetty-launcher-574-thread-1) [n:127.0.0.1:56176_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_2255F3A51CB77A0E-001/cdcr-source-001/node1/.
[junit4] 2> 389967 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[2255F3A51CB77A0E]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 389969 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[2255F3A51CB77A0E]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56174/solr ready
[junit4] 2> 389972 INFO (qtp2082105836-3500) [n:127.0.0.1:56176_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=cdcr-source&name=cdcr-source&nrtReplicas=1&action=CREATE&numShards=1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 389975 INFO (OverseerThreadFactory-1596-thread-1-processing-n:127.0.0.1:56176_solr) [n:127.0.0.1:56176_solr ] o.a.s.c.CreateCollectionCmd Create collection cdcr-source
[junit4] 2> 390090 INFO (OverseerStateUpdate-98528674376843267-127.0.0.1:56176_solr-n_0000000000) [n:127.0.0.1:56176_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-source",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"cdcr-source_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:56176/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 390295 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=cdcr-source&newCollection=true&collection=cdcr-source&version=2&replicaType=NRT&coreNodeName=core_node2&name=cdcr-source_shard1_replica_n1&action=CREATE&numShards=1&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
[junit4] 2> 390295 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 390412 INFO (zkCallback-581-thread-1-processing-n:127.0.0.1:56176_solr) [n:127.0.0.1:56176_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 390412 INFO (zkCallback-581-thread-2-processing-n:127.0.0.1:56176_solr) [n:127.0.0.1:56176_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 391321 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 391326 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
[junit4] 2> 391330 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.IndexSchema [cdcr-source_shard1_replica_n1] Schema name=minimal
[junit4] 2> 391333 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 391335 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Created and persisted managed schema znode at /configs/cdcr-source/managed-schema
[junit4] 2> 391340 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, renamed the non-managed schema /configs/cdcr-source/schema.xml to /configs/cdcr-source/schema.xml.bak
[junit4] 2> 391340 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-source_shard1_replica_n1' using configuration from collection cdcr-source, trusted=true
[junit4] 2> 391340 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56176.solr.core.cdcr-source.shard1.replica_n1' (registry 'solr.core.cdcr-source.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21fd5831
[junit4] 2> 391369 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 391370 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [[cdcr-source_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_2255F3A51CB77A0E-001/cdcr-source-001/node1/cdcr-source_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_2255F3A51CB77A0E-001/cdcr-source-001/node1/./cdcr-source_shard1_replica_n1/data/]
[junit4] 2> 391546 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 391546 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 391551 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 391551 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 391555 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6cdc4ad3[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 391557 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-source
[junit4] 2> 391559 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-source
[junit4] 2> 391559 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-source/managed-schema
[junit4] 2> 391560 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 391561 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 391577 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrBufferStateManager Created znode /collections/cdcr-source/cdcr/state/buffer
[junit4] 2> 391597 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrProcessStateManager Created znode /collections/cdcr-source/cdcr/state/process
[junit4] 2> 391668 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576458792433876992
[junit4] 2> 391669 INFO (searcherExecutor-1599-thread-1-processing-n:127.0.0.1:56176_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@6cdc4ad3[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 391680 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 391680 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 391680 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56176/solr/cdcr-source_shard1_replica_n1/
[junit4] 2> 391680 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 391680 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:56176/solr/cdcr-source_shard1_replica_n1/ has no replicas
[junit4] 2> 391680 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 391688 INFO (zkCallback-581-thread-2-processing-n:127.0.0.1:56176_solr) [n:127.0.0.1:56176_solr ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-source:shard1
[junit4] 2> 391690 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56176/solr/cdcr-source_shard1_replica_n1/ shard1
[junit4] 2> 391804 INFO (zkCallback-581-thread-1-processing-n:127.0.0.1:56176_solr) [n:127.0.0.1:56176_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 391804 INFO (zkCallback-581-thread-2-processing-n:127.0.0.1:56176_solr) [n:127.0.0.1:56176_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 391848 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 391850 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=cdcr-source&newCollection=true&collection=cdcr-source&version=2&replicaType=NRT&coreNodeName=core_node2&name=cdcr-source_shard1_replica_n1&action=CREATE&numShards=1&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=1555
[junit4] 2> 391856 INFO (qtp2082105836-3500) [n:127.0.0.1:56176_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 391966 INFO (zkCallback-581-thread-2-processing-n:127.0.0.1:56176_solr) [n:127.0.0.1:56176_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 391966 INFO (zkCallback-581-thread-1-processing-n:127.0.0.1:56176_solr) [n:127.0.0.1:56176_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 391988 INFO (OverseerCollectionConfigSetProcessor-98528674376843267-127.0.0.1:56176_solr-n_0000000000) [n:127.0.0.1:56176_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 392857 INFO (qtp2082105836-3500) [n:127.0.0.1:56176_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=cdcr-source&name=cdcr-source&nrtReplicas=1&action=CREATE&numShards=1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2} status=0 QTime=2885
[junit4] 1> Adding 10 docs with commit=true, numDocs=100
[junit4] 2> 392874 INFO (qtp2082105836-3503) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=13
[junit4] 2> 392875 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576458793699508224,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 392875 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@8186817 commitCommandVersion:1576458793699508224
[junit4] 2> 392896 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6f121b64[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 392897 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 392898 INFO (searcherExecutor-1599-thread-1-processing-n:127.0.0.1:56176_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@6f121b64[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100)))}
[junit4] 2> 392898 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=23
[junit4] 1> Adding 10 docs with commit=true, numDocs=200
[junit4] 2> 392909 INFO (qtp2082105836-3505) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=9
[junit4] 2> 392910 INFO (qtp2082105836-3498) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576458793736208384,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 392911 INFO (qtp2082105836-3498) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@8186817 commitCommandVersion:1576458793736208384
[junit4] 2> 392928 INFO (qtp2082105836-3498) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6e6c4e75[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 392929 INFO (qtp2082105836-3498) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 392931 INFO (searcherExecutor-1599-thread-1-processing-n:127.0.0.1:56176_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@6e6c4e75[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100)))}
[junit4] 2> 392931 INFO (qtp2082105836-3498) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=20
[junit4] 1> Adding 10 docs with commit=true, numDocs=300
[junit4] 2> 392956 INFO (qtp2082105836-3501) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=23
[junit4] 2> 392957 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576458793785491456,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 392957 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@8186817 commitCommandVersion:1576458793785491456
[junit4] 2> 392976 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@12192c6c[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 392977 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 392978 INFO (searcherExecutor-1599-thread-1-processing-n:127.0.0.1:56176_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@12192c6c[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100)))}
[junit4] 2> 392979 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=22
[junit4] 1> Adding 10 docs with commit=true, numDocs=400
[junit4] 2> 392993 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=13
[junit4] 2> 392995 INFO (qtp2082105836-3500) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576458793825337344,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 392995 INFO (qtp2082105836-3500) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@8186817 commitCommandVersion:1576458793825337344
[junit4] 2> 393050 INFO (qtp2082105836-3500) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2682ade3[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 393051 INFO (qtp2082105836-3500) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 393052 INFO (searcherExecutor-1599-thread-1-processing-n:127.0.0.1:56176_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@2682ade3[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100) Uninverting(_3(7.1.0):C100)))}
[junit4] 2> 393052 INFO (qtp2082105836-3500) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=57
[junit4] 1> Adding 10 docs with commit=true, numDocs=500
[junit4] 2> 393059 INFO (qtp2082105836-3503) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=5
[junit4] 2> 393060 INFO (qtp2082105836-3503) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576458793893494784,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 393060 INFO (qtp2082105836-3503) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@8186817 commitCommandVersion:1576458793893494784
[junit4] 2> 393135 INFO (qtp2082105836-3503) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@50ee1d31[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 393136 INFO (qtp2082105836-3503) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 393138 INFO (searcherExecutor-1599-thread-1-processing-n:127.0.0.1:56176_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@50ee1d31[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100) Uninverting(_3(7.1.0):C100) Uninverting(_4(7.1.0):C100)))}
[junit4] 2> 393138 INFO (qtp2082105836-3503) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=77
[junit4] 1> Adding 10 docs with commit=true, numDocs=600
[junit4] 2> 393146 INFO (qtp2082105836-3505) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=7
[junit4] 2> 393147 INFO (qtp2082105836-3498) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576458793984720896,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 393148 INFO (qtp2082105836-3498) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@8186817 commitCommandVersion:1576458793984720896
[junit4] 2> 393209 INFO (qtp2082105836-3498) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@147e5f54[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 393209 INFO (qtp2082105836-3498) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 393211 INFO (searcherExecutor-1599-thread-1-processing-n:127.0.0.1:56176_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@147e5f54[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100) Uninverting(_3(7.1.0):C100) Uninverting(_4(7.1.0):C100) Uninverting(_5(7.1.0):C100)))}
[junit4] 2> 393212 INFO (qtp2082105836-3498) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=64
[junit4] 1> Adding 10 docs with commit=true, numDocs=700
[junit4] 2> 393221 INFO (qtp2082105836-3501) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=7
[junit4] 2> 393225 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576458794066509824,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 393225 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@8186817 commitCommandVersion:1576458794066509824
[junit4] 2> 393292 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@ffb9fbc[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 393293 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 393295 INFO (searcherExecutor-1599-thread-1-processing-n:127.0.0.1:56176_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@ffb9fbc[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100) Uninverting(_3(7.1.0):C100) Uninverting(_4(7.1.0):C100) Uninverting(_5(7.1.0):C100) Uninverting(_6(7.1.0):C100)))}
[junit4] 2> 393295 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=69
[junit4] 1> Adding 10 docs with commit=true, numDocs=800
[junit4] 2> 393303 INFO (qtp2082105836-3500) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=6
[junit4] 2> 393304 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576458794149347328,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 393304 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@8186817 commitCommandVersion:1576458794149347328
[junit4] 2> 393360 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@55668b3b[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 393360 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 393362 INFO (searcherExecutor-1599-thread-1-processing-n:127.0.0.1:56176_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@55668b3b[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100) Uninverting(_3(7.1.0):C100) Uninverting(_4(7.1.0):C100) Uninverting(_5(7.1.0):C100) Uninverting(_6(7.1.0):C100) Uninverting(_7(7.1.0):C100)))}
[junit4] 2> 393362 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=58
[junit4] 1> Adding 10 docs with commit=true, numDocs=900
[junit4] 2> 393370 INFO (qtp2082105836-3503) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=6
[junit4] 2> 393371 INFO (qtp2082105836-3505) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576458794219601920,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 393371 INFO (qtp2082105836-3505) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@8186817 commitCommandVersion:1576458794219601920
[junit4] 2> 393431 INFO (qtp2082105836-3505) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3e7ada15[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 393432 INFO (qtp2082105836-3505) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 393434 INFO (searcherExecutor-1599-thread-1-processing-n:127.0.0.1:56176_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@3e7ada15[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100) Uninverting(_3(7.1.0):C100) Uninverting(_4(7.1.0):C100) Uninverting(_5(7.1.0):C100) Uninverting(_6(7.1.0):C100) Uninverting(_7(7.1.0):C100) Uninverting(_8(7.1.0):C100)))}
[junit4] 2> 393434 INFO (qtp2082105836-3505) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=63
[junit4] 1> Adding 10 docs with commit=true, numDocs=1000
[junit4] 2> 393441 INFO (qtp2082105836-3498) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=5
[junit4] 2> 393442 INFO (qtp2082105836-3501) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576458794294050816,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 393442 INFO (qtp2082105836-3501) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@8186817 commitCommandVersion:1576458794294050816
[junit4] 2> 393508 INFO (qtp2082105836-3501) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4105cc6f[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 393511 INFO (qtp2082105836-3501) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 393513 INFO (searcherExecutor-1599-thread-1-processing-n:127.0.0.1:56176_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@4105cc6f[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100) Uninverting(_3(7.1.0):C100) Uninverting(_4(7.1.0):C100) Uninverting(_5(7.1.0):C100) Uninverting(_6(7.1.0):C100) Uninverting(_7(7.1.0):C100) Uninverting(_8(7.1.0):C100) Uninverting(_9(7.1.0):C100)))}
[junit4] 2> 393514 INFO (qtp2082105836-3501) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=72
[junit4] 2> 393516 INFO (qtp2082105836-3501) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&_stateVer_=cdcr-source:4&wt=javabin&version=2} hits=1000 status=0 QTime=0
[junit4] 2> 393534 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[2255F3A51CB77A0E]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 393535 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[2255F3A51CB77A0E]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56165/solr ready
[junit4] 2> 393539 INFO (qtp1579143901-3459) [n:127.0.0.1:56167_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=cdcr-target&name=cdcr-target&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 393542 INFO (OverseerThreadFactory-1588-thread-1-processing-n:127.0.0.1:56167_solr) [n:127.0.0.1:56167_solr ] o.a.s.c.CreateCollectionCmd Create collection cdcr-target
[junit4] 2> 393666 INFO (OverseerStateUpdate-98528674332344323-127.0.0.1:56167_solr-n_0000000000) [n:127.0.0.1:56167_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-target",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"cdcr-target_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:56167/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 393883 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=cdcr-target&newCollection=true&name=cdcr-target_shard1_replica_n1&action=CREATE&numShards=1&collection=cdcr-target&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 393885 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 393999 INFO (zkCallback-569-thread-1-processing-n:127.0.0.1:56167_solr) [n:127.0.0.1:56167_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 394000 INFO (zkCallback-569-thread-2-processing-n:127.0.0.1:56167_solr) [n:127.0.0.1:56167_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 394908 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 394911 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
[junit4] 2> 394914 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.IndexSchema [cdcr-target_shard1_replica_n1] Schema name=minimal
[junit4] 2> 394917 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 394918 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Created and persisted managed schema znode at /configs/cdcr-target/managed-schema
[junit4] 2> 394923 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, renamed the non-managed schema /configs/cdcr-target/schema.xml to /configs/cdcr-target/schema.xml.bak
[junit4] 2> 394923 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-target_shard1_replica_n1' using configuration from collection cdcr-target, trusted=true
[junit4] 2> 394923 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56167.solr.core.cdcr-target.shard1.replica_n1' (registry 'solr.core.cdcr-target.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@21fd5831
[junit4] 2> 394924 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 394924 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SolrCore [[cdcr-target_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_2255F3A51CB77A0E-001/cdcr-target-001/node1/cdcr-target_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_2255F3A51CB77A0E-001/cdcr-target-001/node1/./cdcr-target_shard1_replica_n1/data/]
[junit4] 2> 395007 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 395007 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 395028 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 395028 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 395029 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7f11dc98[cdcr-target_shard1_replica_n1] main]
[junit4] 2> 395031 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-target
[junit4] 2> 395032 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-target
[junit4] 2> 395032 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-target/managed-schema
[junit4] 2> 395033 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 395033 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 395038 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.CdcrBufferStateManager Created znode /collections/cdcr-target/cdcr/state/buffer
[junit4] 2> 395039 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.CdcrProcessStateManager Created znode /collections/cdcr-target/cdcr/state/process
[junit4] 2> 395041 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576458795970723840
[junit4] 2> 395042 INFO (searcherExecutor-1605-thread-1-processing-n:127.0.0.1:56167_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-target_shard1_replica_n1] Registered new searcher Searcher@7f11dc98[cdcr-target_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 395049 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 395049 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 395049 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56167/solr/cdcr-target_shard1_replica_n1/
[junit4] 2> 395049 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 395049 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:56167/solr/cdcr-target_shard1_replica_n1/ has no replicas
[junit4] 2> 395049 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 395055 INFO (zkCallback-569-thread-2-processing-n:127.0.0.1:56167_solr) [n:127.0.0.1:56167_solr ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-target:shard1
[junit4] 2> 395057 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56167/solr/cdcr-target_shard1_replica_n1/ shard1
[junit4] 2> 395162 INFO (zkCallback-569-thread-2-processing-n:127.0.0.1:56167_solr) [n:127.0.0.1:56167_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 395162 INFO (zkCallback-569-thread-1-processing-n:127.0.0.1:56167_solr) [n:127.0.0.1:56167_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 395210 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 395212 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=cdcr-target&newCollection=true&name=cdcr-target_shard1_replica_n1&action=CREATE&numShards=1&collection=cdcr-target&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1328
[junit4] 2> 395216 INFO (qtp1579143901-3459) [n:127.0.0.1:56167_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 395317 INFO (zkCallback-569-thread-2-processing-n:127.0.0.1:56167_solr) [n:127.0.0.1:56167_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 395317 INFO (zkCallback-569-thread-1-processing-n:127.0.0.1:56167_solr) [n:127.0.0.1:56167_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 395545 INFO (OverseerCollectionConfigSetProcessor-98528674332344323-127.0.0.1:56167_solr-n_0000000000) [n:127.0.0.1:56167_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 396226 INFO (qtp1579143901-3459) [n:127.0.0.1:56167_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=cdcr-target&name=cdcr-target&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2686
[junit4] 2> 396379 INFO (zkCallback-569-thread-2-processing-n:127.0.0.1:56167_solr) [n:127.0.0.1:56167_solr ] o.a.s.h.CdcrProcessStateManager The CDCR process state has changed: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/cdcr/state/process @ cdcr-target:shard1
[junit4] 2> 396380 INFO (qtp1579143901-3462) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.S.Request [cdcr-target_shard1_replica_n1] webapp=/solr path=/cdcr params={qt=/cdcr&_stateVer_=cdcr-target:4&action=start&wt=javabin&version=2} status=0 QTime=152
[junit4] 2> 396380 INFO (zkCallback-569-thread-2-processing-n:127.0.0.1:56167_solr) [n:127.0.0.1:56167_solr ] o.a.s.h.CdcrProcessStateManager Received new CDCR process state from watcher: STARTED @ cdcr-target:shard1
[junit4] 2> 396385 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 396387 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56165/solr ready
[junit4] 2> 396563 INFO (qtp1579143901-3460) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.CdcrRequestHandler Found maxVersionFromRecent 0 maxVersionFromIndex 0
[junit4] 2> 396563 INFO (qtp1579143901-3460) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.S.Request [cdcr-target_shard1_replica_n1] webapp=/solr path=/cdcr params={action=SHARDCHECKPOINT&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 396564 INFO (qtp1579143901-3464) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.S.Request [cdcr-target_shard1_replica_n1] webapp=/solr path=/cdcr params={_stateVer_=cdcr-target:4&action=COLLECTIONCHECKPOINT&wt=javabin&version=2} status=0 QTime=174
[junit4] 2> 396565 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrReplicatorManager Create new update log reader for target cdcr-target with checkpoint -1 @ cdcr-source:shard1
[junit4] 2> 396565 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrReplicatorManager Attempting to bootstrap target collection: cdcr-target, shard: shard1
[junit4] 2> 396843 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrReplicatorManager Submitting bootstrap task to executor
[junit4] 2> 396845 INFO (cdcr-bootstrap-status-593-thread-1-processing-n:127.0.0.1:56176_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrReplicatorManager Attempting to bootstrap target collection: cdcr-target shard: shard1 leader: http://127.0.0.1:56167/solr/cdcr-target_shard1_replica_n1/
[junit4] 2> 396845 INFO (zkCallback-581-thread-1-processing-n:127.0.0.1:56176_solr) [n:127.0.0.1:56176_solr ] o.a.s.h.CdcrProcessStateManager The CDCR process state has changed: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/cdcr/state/process @ cdcr-source:shard1
[junit4] 2> 396846 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/cdcr params={qt=/cdcr&_stateVer_=cdcr-source:4&action=start&wt=javabin&version=2} status=0 QTime=465
[junit4] 2> 396846 INFO (zkCallback-581-thread-1-processing-n:127.0.0.1:56176_solr) [n:127.0.0.1:56176_solr ] o.a.s.h.CdcrProcessStateManager Received new CDCR process state from watcher: STARTED @ cdcr-source:shard1
[junit4] 2> 396849 INFO (qtp2082105836-3500) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/cdcr params={qt=/cdcr&_stateVer_=cdcr-source:4&action=queues&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 396849 INFO (qtp1579143901-3461) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.S.Request [cdcr-target_shard1_replica_n1] webapp=/solr path=/cdcr params={qt=/cdcr&masterUrl=http://127.0.0.1:56176/solr/cdcr-source_shard1_replica_n1/&action=BOOTSTRAP&wt=javabin&version=2} status=0 QTime=3
[junit4] 1> Cdcr queue response: {responseHeader={status=0,QTime=1},queues={127.0.0.1:56165/solr={cdcr-target={queueSize=1010,lastTimestamp=}}},tlogTotalSize=47659,tlogTotalCount=10,updateLogSynchronizer=stopped}
[junit4] 2> 396853 INFO (qtp1579143901-3462) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576458797870743552,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 396853 INFO (qtp1579143901-3462) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 396854 INFO (qtp1579143901-3462) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 396854 INFO (qtp1579143901-3462) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.S.Request [cdcr-target_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-target:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 396854 INFO (qtp1579143901-3463) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.S.Request [cdcr-target_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&_stateVer_=cdcr-target:4&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 396909 INFO (recoveryExecutor-567-thread-1-processing-n:127.0.0.1:56167_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 396910 INFO (qtp1579143901-3463) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.S.Request [cdcr-target_shard1_replica_n1] webapp=/solr path=/cdcr params={qt=/cdcr&action=BOOTSTRAP_STATUS&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 396911 INFO (cdcr-bootstrap-status-593-thread-1-processing-n:127.0.0.1:56176_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrReplicatorManager CDCR bootstrap running for 1 seconds, sleeping for 2000 ms
[junit4] 2> 396912 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576458797932609536,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 396912 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@8186817 commitCommandVersion:1576458797932609536
[junit4] 2> 396917 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3d7145e7[cdcr-source_shard1_replica_n1] realtime]
[junit4] 2> 396917 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 396917 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&commit_end_point=true&wt=javabin&version=2} status=0 QTime=5
[junit4] 2> 396921 INFO (qtp2082105836-3503) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 396922 INFO (recoveryExecutor-567-thread-1-processing-n:127.0.0.1:56167_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Master's generation: 12
[junit4] 2> 396922 INFO (recoveryExecutor-567-thread-1-processing-n:127.0.0.1:56167_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Master's version: 1503428266461
[junit4] 2> 396922 INFO (recoveryExecutor-567-thread-1-processing-n:127.0.0.1:56167_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 396922 INFO (recoveryExecutor-567-thread-1-processing-n:127.0.0.1:56167_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 396922 INFO (recoveryExecutor-567-thread-1-processing-n:127.0.0.1:56167_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Starting replication process
[junit4] 2> 396928 INFO (qtp2082105836-3503) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.ReplicationHandler Adding tlog files to list: [{size=4657, name=tlog.0000000000000000000.1576458793684828160}, {size=4778, name=tlog.0000000000000000001.1576458793725722624}, {size=4778, name=tlog.0000000000000000002.1576458793759277056}, {size=4778, name=tlog.0000000000000000003.1576458793809608704}, {size=4778, name=tlog.0000000000000000004.1576458793887203328}, {size=4778, name=tlog.0000000000000000005.1576458793976332288}, {size=4778, name=tlog.0000000000000000006.1576458794053926912}, {size=4778, name=tlog.0000000000000000007.1576458794140958720}, {size=4778, name=tlog.0000000000000000008.1576458794211213312}, {size=4778, name=tlog.0000000000000000009.1576458794287759360}]
[junit4] 2> 396928 INFO (qtp2082105836-3503) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&wt=javabin&version=2&command=filelist} status=0 QTime=6
[junit4] 2> 396929 INFO (recoveryExecutor-567-thread-1-processing-n:127.0.0.1:56167_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Number of files in latest index in master: 11
[junit4] 2> 396929 INFO (recoveryExecutor-567-thread-1-processing-n:127.0.0.1:56167_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Number of tlog files in master: 10
[junit4] 2> 396931 INFO (recoveryExecutor-567-thread-1-processing-n:127.0.0.1:56167_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 396931 INFO (recoveryExecutor-567-thread-1-processing-n:127.0.0.1:56167_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Starting download (fullCopy=false) to MockDirectoryWrapper(RAMDirectory@30a83e47 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ff20513)
[junit4] 2> 396932 INFO (qtp2082105836-3501) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a.fnm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 396933 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a.nvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 396935 INFO (qtp2082105836-3500) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a.pos&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 396936 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 396938 INFO (qtp2082105836-3505) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 396940 INFO (qtp2082105836-3498) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 396942 INFO (qtp2082105836-3503) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a.tim&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 396944 INFO (qtp2082105836-3501) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a.tip&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 396948 INFO (qtp2082105836-3502) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 396951 INFO (qtp2082105836-3500) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 396952 INFO (qtp2082105836-3504) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=segments_c&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 396955 INFO (recoveryExecutor-567-thread-1-processing-n:127.0.0.1:56167_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:56167_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Starting download of tlog files from master: [{size=4657, name=tlog.0000000000000000000.1576458793684828160}, {size=4778, name=tlog.0000000000000000001.1576458793725722624}, {size=4778, name=tlog.0000000000000000002.1576458793759277056}, {size=4778, name=tlog.0000000000000000003.1576458793809608704}, {size=4778, name=tlog.0000000000000000004.1576458793887203328}, {size=4778, name=tlog.0000000000000000005.1576458793976332288}, {size=4778, name=tlog.0000000000000000006.1576458794053926912}, {size=4778, name=tlog.0000000000000000007.1576458794140958720}, {size=4778, name=tlog.0000000000000000008.1576458794211213312}, {size=4778, name=tlog.0000000000000000009.1576458794287759360}]
[junit4] 2> 396966 INFO (qtp2082105836-3505) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&checksum=true&tlogFile=tlog.0000000000000000000.1576458793684828160&wt=filestream&command=filecontent} status=0 QTime=6
[junit4] 2> 396968 INFO (qtp2082105836-3498) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&checksum=true&tlogFile=tlog.0000000000000000001.1576458793725722624&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 396970 INFO (qtp2082105836-3503) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&checksum=true&tlogFile=tlog.0000000000000000002.1576458793759277056&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 396973 INFO (qtp2082105836-3501) [n:127.0.0.1:56176_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication
[...truncated too long message...]
unit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {}, locale=es-CR, timezone=Australia/Perth
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=132939408,total=315797504
[junit4] 2> NOTE: All tests run in this JVM: [OpenCloseCoreStressTest, TestUseDocValuesAsStored2, V2ApiIntegrationTest, TestSchemalessBufferedUpdates, SystemInfoHandlerTest, TestCSVLoader, TestTolerantUpdateProcessorCloud, LukeRequestHandlerTest, TestFieldCacheSort, ImplicitSnitchTest, QueryEqualityTest, DistributedFacetPivotLargeTest, CurrencyFieldTypeTest, TestQueryUtils, CoreAdminCreateDiscoverTest, DirectSolrSpellCheckerTest, TestRTimerTree, TestCharFilters, TestSchemaSimilarityResource, HdfsChaosMonkeyNothingIsSafeTest, SolrCmdDistributorTest, TestComplexPhraseLeadingWildcard, TestScoreJoinQPScore, HdfsDirectoryFactoryTest, SOLR749Test, StressHdfsTest, TestMergePolicyConfig, ReplaceNodeTest, JvmMetricsTest, TestZkChroot, CloudMLTQParserTest, TestJsonFacets, DocumentBuilderTest, ShardRoutingCustomTest, TestMissingGroups, TestClassicSimilarityFactory, CleanupOldIndexTest, PreAnalyzedFieldTest, TestCloudSchemaless, PolyFieldTest, TestMiniSolrCloudClusterSSL, CollectionReloadTest, TermVectorComponentTest, TestDistribIDF, TestHdfsUpdateLog, BasicZkTest, TestAnalyzedSuggestions, TestCrossCoreJoin, TestLocalFSCloudBackupRestore, ReplicaListTransformerTest, SpellCheckComponentTest, TestTrie, TestRangeQuery, CdcrBootstrapTest]
[junit4] Completed [131/730 (1!)] on J1 in 159.99s, 3 tests, 1 failure <<< FAILURES!
[...truncated 1565 lines...]
[junit4] Suite: org.apache.solr.metrics.reporters.SolrSlf4jReporterTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.metrics.reporters.SolrSlf4jReporterTest_2255F3A51CB77A0E-001/init-core-data-001
[junit4] 2> 3124091 WARN (SUITE-SolrSlf4jReporterTest-seed#[2255F3A51CB77A0E]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=8 numCloses=8
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=SolrSlf4jReporterTest -Dtests.method=testReporter -Dtests.seed=2255F3A51CB77A0E -Dtests.slow=true -Dtests.locale=es-AR -Dtests.timezone=Africa/Timbuktu -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] FAILURE 5.39s J0 | SolrSlf4jReporterTest.testReporter <<<
[junit4] > Throwable #1: java.lang.AssertionError
[junit4] > at __randomizedtesting.SeedInfo.seed([2255F3A51CB77A0E:7DB1DE9277BBE94B]:0)
[junit4] > at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:74)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.metrics.reporters.SolrSlf4jReporterTest_2255F3A51CB77A0E-001
[junit4] 2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=false): {}, locale=es-AR, timezone=Africa/Timbuktu
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=316399672,total=517357568
[junit4] 2> NOTE: All tests run in this JVM: [BlobRepositoryCloudTest, TestOmitPositions, TestHdfsBackupRestoreCore, TestManagedSchemaThreadSafety, CoreAdminRequestStatusTest, TestSolrCloudSnapshots, TestSolrCloudWithDelegationTokens, TestSolrConfigHandlerCloud, AnalysisErrorHandlingTest, HdfsTlogReplayBufferedWhileIndexingTest, UUIDFieldTest, DistributedDebugComponentTest, BadCopyFieldTest, TestPayloadScoreQParserPlugin, TestReplicationHandler, WordBreakSolrSpellCheckerTest, TestUtils, TestTolerantSearch, TestDynamicFieldResource, ZkControllerTest, TestAtomicUpdateErrorCases, MergeStrategyTest, TestDFISimilarityFactory, AlternateDirectoryTest, CoreSorterTest, HdfsRecoveryZkTest, TestCorePropertiesReload, RemoteQueryErrorTest, MetricsConfigTest, TestSolrCloudWithSecureImpersonation, TestDFRSimilarityFactory, BlockJoinFacetDistribTest, SolrIndexMetricsTest, SaslZkACLProviderTest, TestLFUCache, TestLMDirichletSimilarityFactory, ClusterStateTest, NotRequiredUniqueKeyTest, SpatialRPTFieldTypeTest, MultiTermTest, TestHashPartitioner, ClusterStateUpdateTest, FastVectorHighlighterTest, TestLRUStatsCache, TestIndexingPerformance, CdcrReplicationDistributedZkTest, HdfsDirectoryTest, SolrInfoBeanTest, TestExclusionRuleCollectionAccess, QueryElevationComponentTest, SuggestComponentContextFilterQueryTest, TestSolrCoreProperties, TestReRankQParserPlugin, CdcrRequestHandlerTest, HdfsSyncSliceTest, UtilsToolTest, PeerSyncReplicationTest, NodeMutatorTest, DistributedQueueTest, URLClassifyProcessorTest, TestReqParamsAPI, TestRuleBasedAuthorizationPlugin, TestMinMaxOnMultiValuedField, AnalysisAfterCoreReloadTest, AddSchemaFieldsUpdateProcessorFactoryTest, BitVectorTest, TestTrackingShardHandlerFactory, TestSubQueryTransformerCrossCore, TimeZoneUtilsTest, TestReloadDeadlock, ResponseLogComponentTest, MoveReplicaTest, SolrMetricsIntegrationTest, ReturnFieldsTest, HighlighterMaxOffsetTest, TestTlogReplica, SegmentsInfoRequestHandlerTest, TestPerFieldSimilarityWithDefaultOverride, HLLUtilTest, RequestHandlersTest, TestSystemCollAutoCreate, TestClusterStateMutator, TestShortCircuitedRequests, CollectionStateFormat2Test, TestConfigSetProperties, TestWriterPerf, SortSpecParsingTest, PreAnalyzedFieldManagedSchemaCloudTest, CoreAdminHandlerTest, TestHighlightDedupGrouping, HttpSolrCallGetCoreTest, TestCodecSupport, SolrCoreTest, BaseCdcrDistributedZkTest, TestObjectReleaseTracker, TestConfigsApi, PrimitiveFieldTypeTest, DistanceUnitsTest, TestPHPSerializedResponseWriter, InfixSuggestersTest, TestFieldTypeResource, TestIntervalFaceting, TestFileDictionaryLookup, TestQuerySenderListener, TestSimpleQParserPlugin, BasicDistributedZkTest, NumericFieldsTest, HdfsBasicDistributedZkTest, XmlUpdateRequestHandlerTest, AssignTest, TestSurroundQueryParser, SimplePostToolTest, RAMDirectoryFactoryTest, TestManagedSchema, TestRandomDVFaceting, TestRestManager, TestCoreContainer, DateRangeFieldTest, TestInfoStreamLogging, MigrateRouteKeyTest, BinaryUpdateRequestHandlerTest, DistributedFacetPivotLongTailTest, TestSolrCLIRunExample, TestCloudPivotFacet, LeaderFailoverAfterPartitionTest, TestFunctionQuery, TestUpdate, DataDrivenBlockJoinTest, WrapperMergePolicyFactoryTest, TestDocTermOrds, BlockJoinFacetSimpleTest, SolrTestCaseJ4Test, CachingDirectoryFactoryTest, RequestLoggingTest, BlockDirectoryTest, HardAutoCommitTest, TestSlowCompositeReaderWrapper, TestInitQParser, TestIBSimilarityFactory, SortByFunctionTest, ConcurrentDeleteAndCreateCollectionTest, TestJsonFacetRefinement, CloneFieldUpdateProcessorFactoryTest, SuggesterTest, BigEndianAscendingWordDeserializerTest, TestSort, TestStressVersions, TestXmlQParser, HdfsWriteToMultipleCollectionsTest, TestReversedWildcardFilterFactory, MoveReplicaHDFSTest, TestMultiValuedNumericRangeQuery, SyncSliceTest, ShufflingReplicaListTransformerTest, DirectUpdateHandlerOptimizeTest, TestSQLHandler, HighlighterTest, DeleteStatusTest, TestCollectionsAPIViaSolrCloudCluster, TestPullReplica, TestCollapseQParserPlugin, OutputWriterTest, SolrGraphiteReporterTest, CustomCollectionTest, TestFaceting, TestSchemaManager, SolrIndexSplitterTest, AnalyticsMergeStrategyTest, IndexSchemaTest, TestMacroExpander, DateMathParserTest, TestEmbeddedSolrServerSchemaAPI, TestJsonRequest, RecoveryZkTest, TestFieldCacheWithThreads, ScriptEngineTest, SpatialFilterTest, TestRandomFlRTGCloud, TestDelegationWithHadoopAuth, HighlighterConfigTest, OverseerTest, LeaderElectionIntegrationTest, LeaderElectionTest, ZkSolrClientTest, AutoCommitTest, TestRecovery, DistributedSpellCheckComponentTest, SimpleFacetsTest, StatsComponentTest, BadIndexSchemaTest, ConvertedLegacyTest, TestFiltering, TestLazyCores, SoftAutoCommitTest, QueryParsingTest, PingRequestHandlerTest, TestSolrDeletionPolicy2, TestBinaryField, TestSearchPerf, MinimalSchemaTest, TestFuzzyAnalyzedSuggestions, DirectSolrConnectionTest, TestLuceneMatchVersion, TestPhraseSuggestions, TestXIncludeConfig, EchoParamsTest, TestSweetSpotSimilarityFactory, TestPerFieldSimilarity, ResourceLoaderTest, ChaosMonkeyNothingIsSafeTest, ChaosMonkeySafeLeaderWithPullReplicasTest, CloudExitableDirectoryReaderTest, CollectionsAPIAsyncDistributedZkTest, ConfigSetsAPITest, ConnectionManagerTest, DeleteInactiveReplicaTest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, DocValuesNotIndexedTest, HttpPartitionTest, LeaderInitiatedRecoveryOnCommitTest, MultiThreadedOCPTest, SSLMigrationTest, ShardSplitTest, TestConfigSetsAPIExclusivity, TestConfigSetsAPIZkFailure, TestCryptoKeys, TestDistribDocBasedVersion, TestDownShardTolerantSearch, TestHdfsCloudBackupRestore, TestLeaderElectionWithEmptyReplica, TestLeaderInitiatedRecoveryThread, TestPrepRecovery, TestReplicaProperties, TestRequestForwarding, TestSizeLimitedDistributedMap, TestStressInPlaceUpdates, AutoScalingHandlerTest, TestPolicyCloud, TestDynamicLoading, TestReplicationHandlerBackup, TestRestoreCore, V2StandaloneTest, InfoHandlerTest, MetricsHandlerTest, PropertiesRequestHandlerTest, TestApiFramework, TestCollectionAPIs, TestCoreAdminApis, DistributedFacetExistsSmallTest, DistributedFacetPivotSmallAdvancedTest, DistributedQueryComponentCustomSortTest, DistributedQueryComponentOptimizationTest, SuggestComponentTest, TestHttpShardHandlerFactory, TestUnifiedSolrHighlighter, UninvertDocValuesMergePolicyTest, TestLegacyField, TestLegacyFieldReuse, TestLegacyNumericUtils, TestNumericRangeQuery32, TestNumericRangeQuery64, TestNumericTokenStream, SolrCoreMetricManagerTest, SolrMetricManagerTest, SolrJmxReporterCloudTest, SolrSlf4jReporterTest]
[junit4] Completed [614/730 (2!)] on J0 in 5.47s, 1 test, 1 failure <<< FAILURES!
[...truncated 43724 lines...]