You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2017/08/28 17:23:24 UTC
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 35 - Still
Unstable
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/35/
9 tests failed.
FAILED: org.apache.lucene.spatial3d.TestGeo3DPoint.testGeo3DRelations
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([37424D69F4F2ABF5]:0)
FAILED: org.apache.solr.cloud.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates
Error Message:
Timeout while trying to assert number of documents @ http://127.0.0.1:41220/_uh/qh/source_collection_shard1_replica_n2/
Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ http://127.0.0.1:41220/_uh/qh/source_collection_shard1_replica_n2/
at __randomizedtesting.SeedInfo.seed([A4D131819495438D:77D8619FD106DF1A]:0)
at org.apache.solr.cloud.CdcrReplicationHandlerTest.assertNumDocs(CdcrReplicationHandlerTest.java:256)
at org.apache.solr.cloud.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates(CdcrReplicationHandlerTest.java:236)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.AssertionError: expected:<10> but was:<20>
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.CdcrReplicationHandlerTest.assertNumDocs(CdcrReplicationHandlerTest.java:247)
... 42 more
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([A4D131819495438D:2C850E5B3A692E75]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest: 1) Thread[id=56439, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 2) Thread[id=56438, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest:
1) Thread[id=56439, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
2) Thread[id=56438, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
at __randomizedtesting.SeedInfo.seed([A4D131819495438D]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=56439, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeySafeLeaderTest] at java.lang.Throwable.fillInStackTrace(Native Method) at java.lang.Throwable.fillInStackTrace(Throwable.java:783) at java.lang.Throwable.<init>(Throwable.java:265) at java.lang.Exception.<init>(Exception.java:66) at java.lang.RuntimeException.<init>(RuntimeException.java:62) at java.lang.IllegalStateException.<init>(IllegalStateException.java:55) at org.apache.http.util.Asserts.check(Asserts.java:34) at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:184) at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.requestConnection(PoolingHttpClientConnectionManager.java:251) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:175) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55) at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 2) Thread[id=56438, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest] at java.security.AccessController.doPrivileged(Native Method) at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114) at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99) at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) at java.io.PrintStream.flush(PrintStream.java:338) at java.io.FilterOutputStream.flush(FilterOutputStream.java:140) at java.io.PrintStream.write(PrintStream.java:482) at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291) at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104) at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185) at java.io.PrintStream.newLine(PrintStream.java:546) at java.io.PrintStream.println(PrintStream.java:824) at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748) at java.lang.Throwable.printStackTrace(Throwable.java:658) at java.lang.Throwable.printStackTrace(Throwable.java:643) at java.lang.Throwable.printStackTrace(Throwable.java:634) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=56439, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeySafeLeaderTest]
at java.lang.Throwable.fillInStackTrace(Native Method)
at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
at java.lang.Throwable.<init>(Throwable.java:265)
at java.lang.Exception.<init>(Exception.java:66)
at java.lang.RuntimeException.<init>(RuntimeException.java:62)
at java.lang.IllegalStateException.<init>(IllegalStateException.java:55)
at org.apache.http.util.Asserts.check(Asserts.java:34)
at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:184)
at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.requestConnection(PoolingHttpClientConnectionManager.java:251)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:175)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
2) Thread[id=56438, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
at java.security.AccessController.doPrivileged(Native Method)
at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)
at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
at java.io.PrintStream.flush(PrintStream.java:338)
at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
at java.io.PrintStream.write(PrintStream.java:482)
at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)
at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)
at java.io.PrintStream.newLine(PrintStream.java:546)
at java.io.PrintStream.println(PrintStream.java:824)
at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)
at java.lang.Throwable.printStackTrace(Throwable.java:658)
at java.lang.Throwable.printStackTrace(Throwable.java:643)
at java.lang.Throwable.printStackTrace(Throwable.java:634)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)
at __randomizedtesting.SeedInfo.seed([A4D131819495438D]:0)
FAILED: org.apache.solr.cloud.ShardSplitTest.test
Error Message:
Could not load collection from ZK: collection1
Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
at __randomizedtesting.SeedInfo.seed([A4D131819495438D:2C850E5B3A692E75]:0)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1114)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647)
at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:128)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:154)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:140)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:135)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:902)
at org.apache.solr.cloud.ShardSplitTest.splitByRouteFieldTest(ShardSplitTest.java:691)
at org.apache.solr.cloud.ShardSplitTest.test(ShardSplitTest.java:101)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/state.json
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1126)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1112)
... 49 more
FAILED: org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test
Error Message:
Captured an uncaught exception in thread: Thread[id=3410, name=testExecutor-374-thread-2, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=3410, name=testExecutor-374-thread-2, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
at __randomizedtesting.SeedInfo.seed([A4D131819495438D:2C850E5B3A692E75]:0)
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:33637
at __randomizedtesting.SeedInfo.seed([A4D131819495438D]:0)
at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:595)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:33637
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:638)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCollectionInOneInstance$0(BasicDistributedZkTest.java:593)
... 4 more
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)
... 9 more
FAILED: junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([37424D69F4F2ABF5]:0)
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Timeout waiting for all live and active
Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
at __randomizedtesting.SeedInfo.seed([A4D131819495438D:2C850E5B3A692E75]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 10219 lines...]
[junit4] Suite: org.apache.lucene.spatial3d.TestGeo3DPoint
[junit4] 2> ago 28, 2017 9:04:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2> WARNING: Suite execution timed out: org.apache.lucene.spatial3d.TestGeo3DPoint
[junit4] 2> 1) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
[junit4] 2> 2) Thread[id=12, name=TEST-TestGeo3DPoint.testGeo3DRelations-seed#[37424D69F4F2ABF5], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.testGeo3DRelations(TestGeo3DPoint.java:317)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2> at java.lang.Object.wait(Native Method)
[junit4] 2> at java.lang.Thread.join(Thread.java:1252)
[junit4] 2> at java.lang.Thread.join(Thread.java:1326)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:608)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:457)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
[junit4] 2> 4) Thread[id=11, name=SUITE-TestGeo3DPoint-seed#[37424D69F4F2ABF5], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> at java.lang.Thread.getStackTrace(Thread.java:1559)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
[junit4] 2> at java.security.AccessController.doPrivileged(Native Method)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:678)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:598)
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestGeo3DPoint -Dtests.method=testGeo3DRelations -Dtests.seed=37424D69F4F2ABF5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=es-CL -Dtests.timezone=America/Menominee -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 22.1s J1 | TestGeo3DPoint.testGeo3DRelations <<<
[junit4] > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
[junit4] > at __randomizedtesting.SeedInfo.seed([37424D69F4F2ABF5]:0)
[junit4] 2> ago 28, 2017 9:04:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> ago 28, 2017 9:04:32 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.spatial3d.TestGeo3DPoint:
[junit4] 2> 1) Thread[id=12, name=TEST-TestGeo3DPoint.testGeo3DRelations-seed#[37424D69F4F2ABF5], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoBaseMembershipShape.isWithin(GeoBaseMembershipShape.java:36)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.testGeo3DRelations(TestGeo3DPoint.java:283)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> ago 28, 2017 9:04:32 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=12, name=TEST-TestGeo3DPoint.testGeo3DRelations-seed#[37424D69F4F2ABF5], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> ago 28, 2017 9:04:35 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> SEVERE: There are still zombie threads that couldn't be terminated:
[junit4] 2> 1) Thread[id=12, name=TEST-TestGeo3DPoint.testGeo3DRelations-seed#[37424D69F4F2ABF5], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> at java.lang.Object.hashCode(Native Method)
[junit4] 2> at java.util.WeakHashMap.hash(WeakHashMap.java:298)
[junit4] 2> at java.util.WeakHashMap.getEntry(WeakHashMap.java:426)
[junit4] 2> at java.util.WeakHashMap.containsKey(WeakHashMap.java:417)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:258)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
[junit4] 2> at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:746)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.testGeo3DRelations(TestGeo3DPoint.java:274)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Asserting)}, docValues:{id=DocValuesFormat(name=Direct), point=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1776, maxMBSortInHeap=5.016577876027453, sim=RandomSimilarity(queryNorm=false): {}, locale=es-CL, timezone=America/Menominee
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=69693584,total=497549312
[junit4] 2> NOTE: All tests run in this JVM: [TestGeo3DPoint]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestGeo3DPoint -Dtests.seed=37424D69F4F2ABF5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=es-CL -Dtests.timezone=America/Menominee -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.00s J1 | TestGeo3DPoint (suite) <<<
[junit4] > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
[junit4] > at __randomizedtesting.SeedInfo.seed([37424D69F4F2ABF5]:0)
[junit4] Completed [10/11 (1!)] on J1 in 7225.21s, 7 tests, 2 errors <<< FAILURES!
[...truncated 1813 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/init-core-data-001
[junit4] 2> 346493 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 346493 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 346495 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 346495 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 350734 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 351462 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
[junit4] 2> 351576 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 351833 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 352181 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_lucene2.us.west_apache_org_33969_hdfs____.w0x9h6/webapp
[junit4] 2> 354830 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@lucene2-us-west.apache.org:33969
[junit4] 2> 358127 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 358146 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 358209 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_36777_datanode____m4ojjv/webapp
[junit4] 2> 359558 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36777
[junit4] 2> 360999 WARN (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 361003 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 361150 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_45430_datanode____.pgn1yh/webapp
[junit4] 2> 363500 INFO (SUITE-HdfsUnloadDistributedZkTest-seed#[A4D131819495438D]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45430
[junit4] 2> 369252 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to lucene2-us-west.apache.org/127.0.0.1:34196) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 369254 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to lucene2-us-west.apache.org/127.0.0.1:34196) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 370534 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xe5b70e0ed6e77: from storage DS-bc1a05f6-6b17-4021-95a4-75d67f297135 node DatanodeRegistration(127.0.0.1:38376, datanodeUuid=9b5f89cc-19dd-4229-bd7e-03509dcd9cc9, infoPort=39607, infoSecurePort=0, ipcPort=45701, storageInfo=lv=-56;cid=testClusterID;nsid=1646077084;c=0), blocks: 0, hasStaleStorage: true, processing time: 54 msecs
[junit4] 2> 370551 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xe5b70e1cbe78b: from storage DS-1bbdbdf0-39a2-412c-83de-78e0f07a5722 node DatanodeRegistration(127.0.0.1:32906, datanodeUuid=6748d72e-58dc-436a-9999-affbba7af3d3, infoPort=38852, infoSecurePort=0, ipcPort=40995, storageInfo=lv=-56;cid=testClusterID;nsid=1646077084;c=0), blocks: 0, hasStaleStorage: true, processing time: 16 msecs
[junit4] 2> 370551 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xe5b70e0ed6e77: from storage DS-c6bb5358-3062-4aa3-82c2-b07e34f756a3 node DatanodeRegistration(127.0.0.1:38376, datanodeUuid=9b5f89cc-19dd-4229-bd7e-03509dcd9cc9, infoPort=39607, infoSecurePort=0, ipcPort=45701, storageInfo=lv=-56;cid=testClusterID;nsid=1646077084;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 370551 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xe5b70e1cbe78b: from storage DS-e41833da-b85b-417c-ad14-23eb320a1a3b node DatanodeRegistration(127.0.0.1:32906, datanodeUuid=6748d72e-58dc-436a-9999-affbba7af3d3, infoPort=38852, infoSecurePort=0, ipcPort=40995, storageInfo=lv=-56;cid=testClusterID;nsid=1646077084;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 373277 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 373281 INFO (Thread-267) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 373281 INFO (Thread-267) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 373381 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.ZkTestServer start zk server on port:33225
[junit4] 2> 373428 ERROR (Thread-267) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 373711 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 373741 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 373742 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 373743 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 373744 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 373822 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 373903 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 373904 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 373905 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 373906 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 373923 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 373997 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 375053 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 375194 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2df280ba{/,null,AVAILABLE}
[junit4] 2> 375195 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6f51a86a{HTTP/1.1,[http/1.1]}{127.0.0.1:37022}
[junit4] 2> 375195 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.e.j.s.Server Started @386590ms
[junit4] 2> 375195 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://lucene2-us-west.apache.org:34196/hdfs__lucene2-us-west.apache.org_34196__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.0_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001_tempDir-002_control_data, replicaType=NRT, hostContext=/, hostPort=37022, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/control-001/cores}
[junit4] 2> 375195 ERROR (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 375195 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 375195 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 375195 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 375195 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-28T14:51:25.658Z
[junit4] 2> 375291 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 375292 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/control-001/solr.xml
[junit4] 2> 375386 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@76530e82, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 375429 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33225/solr
[junit4] 2> 376085 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 376086 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37022_
[junit4] 2> 376135 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_ ] o.a.s.c.Overseer Overseer (id=98561679939469316-127.0.0.1:37022_-n_0000000000) starting
[junit4] 2> 376171 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37022_
[junit4] 2> 376354 INFO (zkCallback-210-thread-1-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 377014 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76530e82
[junit4] 2> 377110 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76530e82
[junit4] 2> 377111 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76530e82
[junit4] 2> 377112 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [n:127.0.0.1:37022_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/control-001/cores
[junit4] 2> 377385 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 377385 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33225/solr ready
[junit4] 2> 377698 INFO (qtp2134611594-1578) [n:127.0.0.1:37022_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37022_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 377845 INFO (OverseerThreadFactory-295-thread-1-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 378327 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 378327 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 378615 INFO (zkCallback-210-thread-1-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 379715 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 380039 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 380694 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 380810 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 380810 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76530e82
[junit4] 2> 380912 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://lucene2-us-west.apache.org:34196/solr_hdfs_home
[junit4] 2> 380912 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 380912 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 380912 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://lucene2-us-west.apache.org:34196/solr_hdfs_home/control_collection/core_node1/data/]
[junit4] 2> 380942 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:34196/solr_hdfs_home/control_collection/core_node1/data/snapshot_metadata
[junit4] 2> 381058 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 381058 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 381058 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 383460 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 383492 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:34196/solr_hdfs_home/control_collection/core_node1/data
[junit4] 2> 383669 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:34196/solr_hdfs_home/control_collection/core_node1/data/index
[junit4] 2> 383710 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 383710 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 383710 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 383795 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 383797 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1872306703, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 385109 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:32906 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bc1a05f6-6b17-4021-95a4-75d67f297135:NORMAL:127.0.0.1:38376|RBW], ReplicaUC[[DISK]DS-e41833da-b85b-417c-ad14-23eb320a1a3b:NORMAL:127.0.0.1:32906|RBW]]} size 0
[junit4] 2> 385146 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:38376 is added to blk_1073741825_1001 size 72
[junit4] 2> 385288 WARN (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 385496 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 385496 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 385496 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 385576 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 385576 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 385578 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 385910 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2733c020[control_collection_shard1_replica_n1] main]
[junit4] 2> 385912 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 385912 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 385914 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 385952 INFO (searcherExecutor-298-thread-1-processing-n:127.0.0.1:37022_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@2733c020[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 385952 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576986892215255040
[junit4] 2> 385983 INFO (zkCallback-210-thread-1-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 385984 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 385984 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 385984 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:37022/control_collection_shard1_replica_n1/
[junit4] 2> 385984 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 385984 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:37022/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 385984 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 386003 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:37022/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 386105 INFO (zkCallback-210-thread-1-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 386154 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 386155 INFO (qtp2134611594-1576) [n:127.0.0.1:37022_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=7828
[junit4] 2> 386169 INFO (qtp2134611594-1578) [n:127.0.0.1:37022_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 386260 INFO (zkCallback-210-thread-1-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 387175 INFO (qtp2134611594-1578) [n:127.0.0.1:37022_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37022_&wt=javabin&version=2} status=0 QTime=9477
[junit4] 2> 387248 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 387249 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33225/solr ready
[junit4] 2> 387249 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 387250 INFO (qtp2134611594-1582) [n:127.0.0.1:37022_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 387266 INFO (OverseerThreadFactory-295-thread-2-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_ ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 387266 INFO (OverseerCollectionConfigSetProcessor-98561679939469316-127.0.0.1:37022_-n_0000000000) [n:127.0.0.1:37022_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 387267 WARN (OverseerThreadFactory-295-thread-2-processing-n:127.0.0.1:37022_) [n:127.0.0.1:37022_ ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 387505 INFO (qtp2134611594-1582) [n:127.0.0.1:37022_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 387506 INFO (qtp2134611594-1582) [n:127.0.0.1:37022_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=255
[junit4] 2> 388370 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsUnloadDistributedZkTest_A4D131819495438D-001/shard-1-001 of type NRT
[junit4] 2> 388373 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 388397 INFO (TEST-HdfsUnloadDistributedZkTest.test-seed#[A4D131819495438D]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@67abdb7b{/,null,AVAILABLE}
[junit4] 2> 388397 INFO (TEST-HdfsUnloadDistribut
[...truncated too long message...]
4D131819495438D]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1490372361
[junit4] 2> 4893370 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 4893370 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@234e09e0: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@11e06129
[junit4] 2> 4893390 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 4893390 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4dcf3683: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@3627dfe9
[junit4] 2> 4893402 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 4893402 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@514813b1: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@46799fab
[junit4] 2> 4893402 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 4893434 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [ ] o.a.s.c.Overseer Overseer (id=98561974902784028-127.0.0.1:40483__uh%2Fqh-n_0000000002) closing
[junit4] 2> 4893434 INFO (OverseerStateUpdate-98561974902784028-127.0.0.1:40483__uh%2Fqh-n_0000000002) [n:127.0.0.1:40483__uh%2Fqh ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:40483__uh%2Fqh
[junit4] 2> 4894388 WARN (zkCallback-2004-thread-2-processing-n:127.0.0.1:40483__uh%2Fqh) [n:127.0.0.1:40483__uh%2Fqh c:source_collection s:shard1 r:core_node1 x:source_collection_shard1_replica_n2] o.a.s.c.SyncStrategy Closed, skipping sync up.
[junit4] 2> 4894389 INFO (zkCallback-2004-thread-2-processing-n:127.0.0.1:40483__uh%2Fqh) [n:127.0.0.1:40483__uh%2Fqh c:source_collection s:shard1 r:core_node1 x:source_collection_shard1_replica_n2] o.a.s.c.SolrCore [source_collection_shard1_replica_n2] CLOSING SolrCore org.apache.solr.core.SolrCore@4e086f06
[junit4] 2> 4894389 INFO (zkCallback-2004-thread-2-processing-n:127.0.0.1:40483__uh%2Fqh) [n:127.0.0.1:40483__uh%2Fqh c:source_collection s:shard1 r:core_node1 x:source_collection_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.source_collection.shard1.replica_n2, tag=1309175558
[junit4] 2> 4894389 INFO (zkCallback-2004-thread-2-processing-n:127.0.0.1:40483__uh%2Fqh) [n:127.0.0.1:40483__uh%2Fqh c:source_collection s:shard1 r:core_node1 x:source_collection_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1f0ef195: rootName = null, domain = solr.core.source_collection.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.source_collection.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@66e1c78d
[junit4] 2> 4894417 INFO (zkCallback-2004-thread-2-processing-n:127.0.0.1:40483__uh%2Fqh) [n:127.0.0.1:40483__uh%2Fqh c:source_collection s:shard1 r:core_node1 x:source_collection_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.source_collection.shard1.leader, tag=1309175558
[junit4] 2> 4894429 INFO (zkCallback-2004-thread-2-processing-n:127.0.0.1:40483__uh%2Fqh) [n:127.0.0.1:40483__uh%2Fqh c:source_collection s:shard1 r:core_node1 x:source_collection_shard1_replica_n2] o.a.s.h.CdcrRequestHandler Solr core is being closed - shutting down CDCR handler @ source_collection:shard1
[junit4] 2> 4894433 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@5bb2f8b1{HTTP/1.1,[http/1.1]}{127.0.0.1:40483}
[junit4] 2> 4894433 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7b27c0ca{/_uh/qh,null,UNAVAILABLE}
[junit4] 2> 4894441 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testPartialReplication
[junit4] 2> 4894441 ERROR (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 4894442 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[A4D131819495438D]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37066 37066
[junit4] 2> 4894578 INFO (Thread-26610) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37066 37066
[junit4] 2> 4894578 WARN (Thread-26610) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 22 /solr/aliases.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2> 4 /solr/collections/source_collection/leaders/shard1/leader
[junit4] 2> 3 /solr/security.json
[junit4] 2> 2 /solr/collections/tmp_collection/leaders/shard1/leader
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 22 /solr/clusterstate.json
[junit4] 2> 22 /solr/clusterprops.json
[junit4] 2> 4 /solr/collections/source_collection/cdcr/state/buffer
[junit4] 2> 4 /solr/collections/source_collection/cdcr/state/process
[junit4] 2> 3 /solr/collections/source_collection/state.json
[junit4] 2> 2 /solr/collections/tmp_collection/state.json
[junit4] 2> 2 /solr/overseer_elect/election/98561974902784004-127.0.0.1:41887__uh%2Fqh-n_0000000000
[junit4] 2> 2 /solr/collections/source_collection/leader_elect/shard1/election/98561974902784004-core_node2-n_0000000000
[junit4] 2> 2 /solr/collections/tmp_collection/cdcr/state/buffer
[junit4] 2> 2 /solr/collections/tmp_collection/cdcr/state/process
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 22 /solr/collections
[junit4] 2> 21 /solr/live_nodes
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/overseer/collection-queue-work
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CdcrReplicationHandlerTest_A4D131819495438D-001
[junit4] 2> Aug 28, 2017 4:06:44 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=FSTOrd50}, docValues:{_version_=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=918, maxMBSortInHeap=6.724546786203957, sim=RandomSimilarity(queryNorm=true): {}, locale=pt-BR, timezone=Europe/Bratislava
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=171909696,total=525336576
[junit4] 2> NOTE: All tests run in this JVM: [DirectoryFactoryTest, DeleteReplicaTest, V2ApiIntegrationTest, TestGraphMLResponseWriter, ShardSplitTest, TestFieldCacheSort, DistributedDebugComponentTest, TestFoldingMultitermQuery, TestCloudPseudoReturnFields, HdfsChaosMonkeyNothingIsSafeTest, BadIndexSchemaTest, TestRestoreCore, TestHighFrequencyDictionaryFactory, TestTolerantSearch, TestChildDocTransformer, ConjunctionSolrSpellCheckerTest, DeleteInactiveReplicaTest, CloneFieldUpdateProcessorFactoryTest, ReplaceNodeTest, JSONWriterTest, TestIntervalFaceting, MultiTermTest, TestFieldCollectionResource, TestStressReorder, TestStressInPlaceUpdates, SearchHandlerTest, TestOnReconnectListenerSupport, HdfsRestartWhileUpdatingTest, TestRestManager, ExplicitHLLTest, TestExactStatsCache, TestDistribDocBasedVersion, PrimitiveFieldTypeTest, DocValuesMissingTest, DateFieldTest, LeaderFailoverAfterPartitionTest, DistributedQueueTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestPointFields, SpatialHeatmapFacetsTest, TestDocTermOrds, TestNonDefinedSimilarityFactory, SimpleCollectionCreateDeleteTest, DocumentBuilderTest, HdfsNNFailoverTest, SolrJmxReporterTest, MinimalSchemaTest, StatsComponentTest, TestRawResponseWriter, DistributedFacetPivotSmallTest, SearchHandlerTest, ConnectionManagerTest, FullSolrCloudDistribCmdsTest, HdfsTlogReplayBufferedWhileIndexingTest, DistribCursorPagingTest, TestDelegationWithHadoopAuth, LeaderElectionTest, TestFieldCacheWithThreads, ChangedSchemaMergeTest, RegexBoostProcessorTest, RollingRestartTest, CleanupOldIndexTest, DirectUpdateHandlerOptimizeTest, OverriddenZkACLAndCredentialsProvidersTest, TestDFRSimilarityFactory, DateMathParserTest, CollectionStateFormat2Test, CdcrReplicationHandlerTest]
[junit4] Completed [676/729 (5!)] on J2 in 756.95s, 5 tests, 1 failure <<< FAILURES!
[...truncated 43486 lines...]
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 43 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/43/
6 tests failed.
FAILED: org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([36541951E3BFD641]:0)
FAILED: junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([36541951E3BFD641]:0)
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([45527DBA3A5217C7:CD06426094AE7A3F]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:174)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest: 1) Thread[id=54112, name=StoppableIndexingThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 2) Thread[id=54056, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 3) Thread[id=54218, name=zkCallback-3929-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=54057, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 5) Thread[id=54219, name=zkCallback-3929-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=54111, name=StoppableIndexingThread-SendThread(127.0.0.1:44864), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 7) Thread[id=54058, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55) 8) Thread[id=54113, name=zkCallback-3929-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:
1) Thread[id=54112, name=StoppableIndexingThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
2) Thread[id=54056, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
3) Thread[id=54218, name=zkCallback-3929-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=54057, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
5) Thread[id=54219, name=zkCallback-3929-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=54111, name=StoppableIndexingThread-SendThread(127.0.0.1:44864), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
7) Thread[id=54058, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)
8) Thread[id=54113, name=zkCallback-3929-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([45527DBA3A5217C7]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=54056, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 2) Thread[id=54218, name=zkCallback-3929-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=54057, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 4) Thread[id=54219, name=zkCallback-3929-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=54111, name=StoppableIndexingThread-SendThread(127.0.0.1:44864), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 6) Thread[id=54113, name=zkCallback-3929-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=54056, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
2) Thread[id=54218, name=zkCallback-3929-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=54057, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
4) Thread[id=54219, name=zkCallback-3929-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=54111, name=StoppableIndexingThread-SendThread(127.0.0.1:44864), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
6) Thread[id=54113, name=zkCallback-3929-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([45527DBA3A5217C7]:0)
FAILED: org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload
Error Message:
expected:<[{indexVersion=1505272155843,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _6.cfe, _6.cfs, _6.si, _7.cfe, _7.cfs, _7.si, segments_2]}]> but was:<[{indexVersion=1505272155843,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _6.cfe, _6.cfs, _6.si, _7.cfe, _7.cfs, _7.si, segments_2]}, {indexVersion=1505272155843,generation=3,filelist=[_8.cfe, _8.cfs, _8.si, segments_3]}]>
Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1505272155843,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _6.cfe, _6.cfs, _6.si, _7.cfe, _7.cfs, _7.si, segments_2]}]> but was:<[{indexVersion=1505272155843,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _6.cfe, _6.cfs, _6.si, _7.cfe, _7.cfs, _7.si, segments_2]}, {indexVersion=1505272155843,generation=3,filelist=[_8.cfe, _8.cfs, _8.si, segments_3]}]>
at __randomizedtesting.SeedInfo.seed([45527DBA3A5217C7:6085668A4A1A19C4]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1277)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 10241 lines...]
[junit4] Suite: org.apache.lucene.spatial3d.TestGeo3DPoint
[junit4] 2> сеп 13, 2017 3:18:04 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2> WARNING: Suite execution timed out: org.apache.lucene.spatial3d.TestGeo3DPoint
[junit4] 2> 1) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
[junit4] 2> 2) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2> at java.lang.Object.wait(Native Method)
[junit4] 2> at java.lang.Thread.join(Thread.java:1252)
[junit4] 2> at java.lang.Thread.join(Thread.java:1326)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:608)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:457)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
[junit4] 2> 3) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[36541951E3BFD641], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.isWithin(GeoComplexPolygon.java:326)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoBaseMembershipShape.isWithin(GeoBaseMembershipShape.java:36)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoBaseShape.getBounds(GeoBaseShape.java:49)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.getBounds(GeoComplexPolygon.java:375)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 4) Thread[id=11, name=SUITE-TestGeo3DPoint-seed#[36541951E3BFD641], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> at java.lang.Thread.getStackTrace(Thread.java:1559)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
[junit4] 2> at java.security.AccessController.doPrivileged(Native Method)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:678)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:598)
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestGeo3DPoint -Dtests.method=testRandomBig -Dtests.seed=36541951E3BFD641 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=sr-ME -Dtests.timezone=Africa/Tripoli -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 6783s J2 | TestGeo3DPoint.testRandomBig <<<
[junit4] > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
[junit4] > at __randomizedtesting.SeedInfo.seed([36541951E3BFD641]:0)
[junit4] 2> сеп 13, 2017 3:18:05 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> сеп 13, 2017 3:18:25 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.spatial3d.TestGeo3DPoint:
[junit4] 2> 1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[36541951E3BFD641], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> at org.apache.lucene.store.MockIndexInputWrapper.readShort(MockIndexInputWrapper.java:159)
[junit4] 2> at org.apache.lucene.store.IndexInput$1.readShort(IndexInput.java:138)
[junit4] 2> at org.apache.lucene.util.packed.DirectReader$DirectPackedReader16.get(DirectReader.java:185)
[junit4] 2> at org.apache.lucene.codecs.lucene70.Lucene70DocValuesProducer$2.longValue(Lucene70DocValuesProducer.java:471)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:840)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> сеп 13, 2017 3:18:25 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[36541951E3BFD641], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> сеп 13, 2017 3:18:28 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> SEVERE: There are still zombie threads that couldn't be terminated:
[junit4] 2> 1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[36541951E3BFD641], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> at org.apache.lucene.spatial3d.geom.Plane.findIntersectionBounds(Plane.java:1097)
[junit4] 2> at org.apache.lucene.spatial3d.geom.Plane.recordBounds(Plane.java:1214)
[junit4] 2> at org.apache.lucene.spatial3d.geom.XYZBounds.addIntersection(XYZBounds.java:262)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoConcavePolygon.getBounds(GeoConcavePolygon.java:439)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoCompositeMembershipShape.getBounds(GeoCompositeMembershipShape.java:75)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Memory)}, docValues:{id=DocValuesFormat(name=Lucene70), point=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=271, maxMBSortInHeap=5.640720867930634, sim=RandomSimilarity(queryNorm=true): {}, locale=sr-ME, timezone=Africa/Tripoli
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=242358640,total=475529216
[junit4] 2> NOTE: All tests run in this JVM: [TestGeo3DPoint]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestGeo3DPoint -Dtests.seed=36541951E3BFD641 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=sr-ME -Dtests.timezone=Africa/Tripoli -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J2 | TestGeo3DPoint (suite) <<<
[junit4] > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
[junit4] > at __randomizedtesting.SeedInfo.seed([36541951E3BFD641]:0)
[junit4] Completed [11/11 (1!)] on J2 in 7224.55s, 11 tests, 2 errors <<< FAILURES!
[...truncated 2806 lines...]
[junit4] Suite: org.apache.solr.handler.TestReplicationHandler
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/init-core-data-001
[junit4] 2> 5762760 WARN (SUITE-TestReplicationHandler-seed#[45527DBA3A5217C7]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=74 numCloses=74
[junit4] 2> 5762760 INFO (SUITE-TestReplicationHandler-seed#[45527DBA3A5217C7]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 5762761 INFO (SUITE-TestReplicationHandler-seed#[45527DBA3A5217C7]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
[junit4] 2> 5762762 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.SolrTestCaseJ4 ###Starting doTestStopPoll
[junit4] 2> 5762762 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-001/collection1
[junit4] 2> 5762781 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 5762812 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@63e1d34f{/solr,null,AVAILABLE}
[junit4] 2> 5762812 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1e175c07{HTTP/1.1,[http/1.1]}{127.0.0.1:44514}
[junit4] 2> 5762824 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.e.j.s.Server Started @5778059ms
[junit4] 2> 5762824 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=44514}
[junit4] 2> 5762825 ERROR (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5762825 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 5762825 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in standalone mode on port null
[junit4] 2> 5762825 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5762825 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-13T03:03:55.458Z
[junit4] 2> 5762825 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-001/solr.xml
[junit4] 2> 5762855 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@68dbcd60, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5763053 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68dbcd60
[junit4] 2> 5763095 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68dbcd60
[junit4] 2> 5763095 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68dbcd60
[junit4] 2> 5763133 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-001/.
[junit4] 2> 5763133 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 5763330 INFO (coreLoadExecutor-7442-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 5763367 INFO (coreLoadExecutor-7442-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 5763369 INFO (coreLoadExecutor-7442-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
[junit4] 2> 5763369 INFO (coreLoadExecutor-7442-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-001/./collection1, trusted=true
[junit4] 2> 5763385 INFO (coreLoadExecutor-7442-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68dbcd60
[junit4] 2> 5763386 INFO (coreLoadExecutor-7442-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5763386 INFO (coreLoadExecutor-7442-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-001/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-001/./collection1/data/]
[junit4] 2> 5763387 INFO (coreLoadExecutor-7442-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=43.126953125, floorSegmentMB=0.7705078125, forceMergeDeletesPctAllowed=24.60716279491469, segmentsPerTier=3.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.28720781238189314
[junit4] 2> 5763533 INFO (coreLoadExecutor-7442-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5763533 INFO (coreLoadExecutor-7442-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5763534 INFO (coreLoadExecutor-7442-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@7ca74773
[junit4] 2> 5763534 INFO (coreLoadExecutor-7442-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6d3eb122[collection1] main]
[junit4] 2> 5763535 INFO (coreLoadExecutor-7442-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-001/collection1/conf
[junit4] 2> 5763535 INFO (coreLoadExecutor-7442-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml,xslt/dummy.xsl
[junit4] 2> 5763535 INFO (coreLoadExecutor-7442-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 5763549 INFO (searcherExecutor-7443-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6d3eb122[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5763794 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-002/collection1
[junit4] 2> 5763809 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 5763810 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1fdfbd77{/solr,null,AVAILABLE}
[junit4] 2> 5763810 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@12e039cf{HTTP/1.1,[http/1.1]}{127.0.0.1:35161}
[junit4] 2> 5763810 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.e.j.s.Server Started @5779044ms
[junit4] 2> 5763810 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-002/collection1/data, hostContext=/solr, hostPort=35161}
[junit4] 2> 5763810 ERROR (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5763810 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 5763810 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in standalone mode on port null
[junit4] 2> 5763810 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5763810 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-13T03:03:56.443Z
[junit4] 2> 5763811 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-002/solr.xml
[junit4] 2> 5763936 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@68dbcd60, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5764125 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68dbcd60
[junit4] 2> 5764181 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68dbcd60
[junit4] 2> 5764181 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68dbcd60
[junit4] 2> 5764195 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-002/.
[junit4] 2> 5764195 INFO (TEST-TestReplicationHandler.doTestStopPoll-seed#[45527DBA3A5217C7]) [ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 5764300 INFO (coreLoadExecutor-7452-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 5764359 INFO (coreLoadExecutor-7452-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 5764361 INFO (coreLoadExecutor-7452-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
[junit4] 2> 5764361 INFO (coreLoadExecutor-7452-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-002/./collection1, trusted=true
[junit4] 2> 5764373 INFO (coreLoadExecutor-7452-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68dbcd60
[junit4] 2> 5764373 INFO (coreLoadExecutor-7452-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5764373 INFO (coreLoadExecutor-7452-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-002/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-002/./collection1/data/]
[junit4] 2> 5764375 INFO (coreLoadExecutor-7452-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=43.126953125, floorSegmentMB=0.7705078125, forceMergeDeletesPctAllowed=24.60716279491469, segmentsPerTier=3.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.28720781238189314
[junit4] 2> 5764612 INFO (coreLoadExecutor-7452-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5764613 INFO (coreLoadExecutor-7452-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5764614 INFO (coreLoadExecutor-7452-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@239466b1
[junit4] 2> 5764614 INFO (coreLoadExecutor-7452-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7f134e15[collection1] main]
[junit4] 2> 5764614 INFO (coreLoadExecutor-7452-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_45527DBA3A5217C7-001/solr-instance-002/collection1/conf
[junit4] 2> 5764629 INFO (coreLoadExecutor-7452-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 1000ms
[junit4] 2> 5764650 INFO (searcherExecutor-7453-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7f134e15[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5764703 INFO (qtp1176835690-165123) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 5764703 INFO (indexFetcher-7457-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 5764703 INFO (indexFetcher-7457-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 5764703 INFO (indexFetcher-7457-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 5764703 INFO (indexFetcher-7457-thread-1) [ x:collection1] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 5764983 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 5764985 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[0]} 0 1
[junit4] 2> 5764985 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1]} 0 0
[junit4] 2> 5764985 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2]} 0 0
[junit4] 2> 5764986 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[3]} 0 0
[junit4] 2> 5765002 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[4]} 0 16
[junit4] 2> 5765003 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[5]} 0 0
[junit4] 2> 5765003 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[6]} 0 0
[junit4] 2> 5765003 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[7]} 0 0
[junit4] 2> 5765003 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[8]} 0 0
[junit4] 2> 5765004 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[9]} 0 0
[junit4] 2> 5765004 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[10]} 0 0
[junit4] 2> 5765004 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[11]} 0 0
[junit4] 2> 5765004 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[12]} 0 0
[junit4] 2> 5765005 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[13]} 0 0
[junit4] 2> 5765005 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[14]} 0 0
[junit4] 2> 5765005 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[15]} 0 0
[junit4] 2> 5765005 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[16]} 0 0
[junit4] 2> 5765006 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[17]} 0 0
[junit4] 2> 5765006 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[18]} 0 0
[junit4] 2> 5765006 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[19]} 0 0
[junit4] 2> 5765006 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[20]} 0 0
[junit4] 2> 5765020 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[21]} 0 0
[junit4] 2> 5765035 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[22]} 0 0
[junit4] 2> 5765036 INFO (qtp1176835690-165126) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[23]} 0 0
[junit4] 2> 5765036 INFO (qtp1176835690-165127) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[24]} 0 0
[junit4] 2> 5765036 INFO (qtp1176835690-165120) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[25]} 0 0
[junit4] 2> 5765037 INFO (qtp1176835690-165124) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[26]} 0 0
[junit4] 2> 5765051 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[27]} 0 0
[junit4] 2> 5765052 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[28]} 0 0
[junit4] 2> 5765052 INFO (qtp1176835690-165126) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[29]} 0 0
[junit4] 2> 5765052 INFO (qtp1176835690-165126) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[30]} 0 0
[junit4] 2> 5765052 INFO (qtp1176835690-165120) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[31]} 0 0
[junit4] 2> 5765053 INFO (qtp1176835690-165124) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[32]} 0 0
[junit4] 2> 5765053 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[33]} 0 0
[junit4] 2> 5765053 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[34]} 0 0
[junit4] 2> 5765053 INFO (qtp1176835690-165127) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[35]} 0 0
[junit4] 2> 5765054 INFO (qtp1176835690-165127) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[36]} 0 0
[junit4] 2> 5765054 INFO (qtp1176835690-165127) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[37]} 0 0
[junit4] 2> 5765054 INFO (qtp1176835690-165127) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[38]} 0 0
[junit4] 2> 5765054 INFO (qtp1176835690-165121) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[39]} 0 0
[junit4] 2> 5765055 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[40]} 0 0
[junit4] 2> 5765055 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[41]} 0 0
[junit4] 2> 5765055 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[42]} 0 0
[junit4] 2> 5765055 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[43]} 0 0
[junit4] 2> 5765056 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[44]} 0 0
[junit4] 2> 5765056 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[45]} 0 0
[junit4] 2> 5765056 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[46]} 0 0
[junit4] 2> 5765057 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[47]} 0 0
[junit4] 2> 5765057 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[48]} 0 0
[junit4] 2> 5765057 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[49]} 0 0
[junit4] 2> 5765057 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[50]} 0 0
[junit4] 2> 5765058 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[51]} 0 0
[junit4] 2> 5765058 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[52]} 0 0
[junit4] 2> 5765058 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[53]} 0 0
[junit4] 2> 5765058 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[54]} 0 0
[junit4] 2> 5765059 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[55]} 0 0
[junit4] 2> 5765059 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[56]} 0 0
[junit4] 2> 5765076 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[57]} 0 0
[junit4] 2> 5765076 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[58]} 0 0
[junit4] 2> 5765076 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[59]} 0 0
[junit4] 2> 5765076 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[60]} 0 0
[junit4] 2> 5765077 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[61]} 0 0
[junit4] 2> 5765077 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[62]} 0 0
[junit4] 2> 5765077 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[63]} 0 0
[junit4] 2> 5765077 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[64]} 0 0
[junit4] 2> 5765078 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[65]} 0 0
[junit4] 2> 5765078 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[66]} 0 0
[junit4] 2> 5765078 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[67]} 0 0
[junit4] 2> 5765078 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[68]} 0 0
[junit4] 2> 5765079 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[69]} 0 0
[junit4] 2> 5765079 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[70]} 0 0
[junit4] 2> 5765079 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[71]} 0 0
[junit4] 2> 5765079 INFO (qtp1176835690-165125) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[72]} 0 0
[junit4] 2> 5765092 INFO (qtp1176835690-165126) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[73]} 0 0
[junit4] 2> 5765092 INFO (qtp1176835690-165126) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[74]} 0 0
[junit4] 2> 5765093 INFO (qtp1176835690-165126) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[75]} 0 0
[junit4] 2> 5765093 INFO (qtp1176835690-165126) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[76]} 0 0
[junit4] 2> 5765093 INFO (qtp1176835690-165126) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] w
[...truncated too long message...]
)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([45527DBA3A5217C7]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=54056, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
[junit4] > at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
[junit4] > at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
[junit4] > 2) Thread[id=54218, name=zkCallback-3929-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 3) Thread[id=54057, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
[junit4] > at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
[junit4] > at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
[junit4] > 4) Thread[id=54219, name=zkCallback-3929-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=54111, name=StoppableIndexingThread-SendThread(127.0.0.1:44864), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
[junit4] > 6) Thread[id=54113, name=zkCallback-3929-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([45527DBA3A5217C7]:0)
[junit4] Completed [504/729 (2!)] on J2 in 174.73s, 1 test, 1 failure, 2 errors <<< FAILURES!
[...truncated 43981 lines...]
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 42 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/42/
3 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
Error Message:
8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 1) Thread[id=54844, name=zkCallback-2419-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=54898, name=zkCallback-2419-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=54380, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]-SendThread(127.0.0.1:57190), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230) at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1246) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1170) 4) Thread[id=54990, name=zkCallback-2419-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=54382, name=zkCallback-2419-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=54379, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=54843, name=zkCallback-2419-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=54381, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:
1) Thread[id=54844, name=zkCallback-2419-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=54898, name=zkCallback-2419-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=54380, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]-SendThread(127.0.0.1:57190), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230)
at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1246)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1170)
4) Thread[id=54990, name=zkCallback-2419-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=54382, name=zkCallback-2419-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=54379, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=54843, name=zkCallback-2419-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=54381, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
at __randomizedtesting.SeedInfo.seed([FB496A7D645EB255]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=54844, name=zkCallback-2419-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=54898, name=zkCallback-2419-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=54380, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]-SendThread(127.0.0.1:57190), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 4) Thread[id=54990, name=zkCallback-2419-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=54382, name=zkCallback-2419-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=54843, name=zkCallback-2419-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=54844, name=zkCallback-2419-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=54898, name=zkCallback-2419-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=54380, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]-SendThread(127.0.0.1:57190), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
4) Thread[id=54990, name=zkCallback-2419-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=54382, name=zkCallback-2419-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=54843, name=zkCallback-2419-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([FB496A7D645EB255]:0)
FAILED: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test
Error Message:
document count mismatch. control=115 sum(shards)=111 cloudClient=111
Stack Trace:
java.lang.AssertionError: document count mismatch. control=115 sum(shards)=111 cloudClient=111
at __randomizedtesting.SeedInfo.seed([FB496A7D645EB255:731D55A7CAA2DFAD]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1375)
at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:235)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13291 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/init-core-data-001
[junit4] 2> 5409478 INFO (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[FB496A7D645EB255]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 5409479 INFO (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[FB496A7D645EB255]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 5409479 INFO (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[FB496A7D645EB255]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /dci/fs
[junit4] 2> 5409487 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 5409494 INFO (Thread-40013) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 5409494 INFO (Thread-40013) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 5409514 ERROR (Thread-40013) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 5409597 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.ZkTestServer start zk server on port:57190
[junit4] 2> 5409838 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 5409839 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 5409840 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 5409841 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 5409841 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 5409842 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 5409842 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 5409843 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 5409843 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 5409844 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 5409845 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 5409845 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 5410488 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 5410493 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@27423f96{/dci/fs,null,AVAILABLE}
[junit4] 2> 5410493 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@742bc9e0{HTTP/1.1,[http/1.1]}{127.0.0.1:38119}
[junit4] 2> 5410493 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.Server Started @5422637ms
[junit4] 2> 5410493 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/tempDir-001/control/data, hostContext=/dci/fs, hostPort=38119, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/control-001/cores}
[junit4] 2> 5410514 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5410515 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 5410515 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5410515 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5410515 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-11T10:33:13.661Z
[junit4] 2> 5410524 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5410525 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/control-001/solr.xml
[junit4] 2> 5410551 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5410563 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57190/solr
[junit4] 2> 5410713 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5410818 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38119_dci%2Ffs
[junit4] 2> 5410820 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.Overseer Overseer (id=98639937054179332-127.0.0.1:38119_dci%2Ffs-n_0000000000) starting
[junit4] 2> 5410862 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38119_dci%2Ffs
[junit4] 2> 5410874 INFO (OverseerStateUpdate-98639937054179332-127.0.0.1:38119_dci%2Ffs-n_0000000000) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5411255 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8
[junit4] 2> 5411293 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8
[junit4] 2> 5411293 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8
[junit4] 2> 5411295 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/control-001/cores
[junit4] 2> 5411926 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5411927 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57190/solr ready
[junit4] 2> 5411942 INFO (qtp2108176362-54347) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38119_dci%252Ffs&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5411965 INFO (OverseerThreadFactory-8020-thread-1-processing-n:127.0.0.1:38119_dci%2Ffs) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 5412087 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 5412087 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 5412190 INFO (zkCallback-2412-thread-1-processing-n:127.0.0.1:38119_dci%2Ffs) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 5413111 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 5413575 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 5413957 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 5414008 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 5414009 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8
[junit4] 2> 5414009 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5414010 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 5414021 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=40, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.96484375, noCFSRatio=0.0]
[junit4] 2> 5414037 WARN (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 5414214 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5414215 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5414216 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 5414216 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5414217 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=27, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 5414223 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@63b59b0[control_collection_shard1_replica_n1] main]
[junit4] 2> 5414226 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 5414226 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 5414227 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 5414229 INFO (searcherExecutor-8023-thread-1-processing-n:127.0.0.1:38119_dci%2Ffs x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@63b59b0[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5414230 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578238997824536576
[junit4] 2> 5414241 INFO (zkCallback-2412-thread-1-processing-n:127.0.0.1:38119_dci%2Ffs) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 5414242 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5414242 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 5414242 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38119/dci/fs/control_collection_shard1_replica_n1/
[junit4] 2> 5414242 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 5414242 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:38119/dci/fs/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 5414242 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 5414246 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38119/dci/fs/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 5414480 INFO (zkCallback-2412-thread-1-processing-n:127.0.0.1:38119_dci%2Ffs) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 5414530 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 5414534 INFO (qtp2108176362-54345) [n:127.0.0.1:38119_dci%2Ffs c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2447
[junit4] 2> 5414537 INFO (qtp2108176362-54347) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 5414642 INFO (zkCallback-2412-thread-1-processing-n:127.0.0.1:38119_dci%2Ffs) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 5415537 INFO (qtp2108176362-54347) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38119_dci%252Ffs&wt=javabin&version=2} status=0 QTime=3595
[junit4] 2> 5415574 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5415575 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57190/solr ready
[junit4] 2> 5415575 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 5415576 INFO (qtp2108176362-54348) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5415577 INFO (OverseerCollectionConfigSetProcessor-98639937054179332-127.0.0.1:38119_dci%2Ffs-n_0000000000) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5415578 INFO (OverseerThreadFactory-8020-thread-2-processing-n:127.0.0.1:38119_dci%2Ffs) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 5415578 WARN (OverseerThreadFactory-8020-thread-2-processing-n:127.0.0.1:38119_dci%2Ffs) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 5415830 INFO (qtp2108176362-54348) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 5415830 INFO (qtp2108176362-54348) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=254
[junit4] 2> 5416541 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-1-001 of type NRT
[junit4] 2> 5416541 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 5416596 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2391ad88{/dci/fs,null,AVAILABLE}
[junit4] 2> 5416597 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1e687ae4{HTTP/1.1,[http/1.1]}{127.0.0.1:59392}
[junit4] 2> 5416597 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.Server Started @5428740ms
[junit4] 2> 5416597 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/dci/fs, hostPort=59392, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-1-001/cores}
[junit4] 2> 5416597 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5416597 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 5416597 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5416597 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5416597 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-11T10:33:19.743Z
[junit4] 2> 5416623 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5416623 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-1-001/solr.xml
[junit4] 2> 5416628 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5416641 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57190/solr
[junit4] 2> 5416681 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:59392_dci%2Ffs ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5416682 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:59392_dci%2Ffs ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5416683 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:59392_dci%2Ffs ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59392_dci%2Ffs
[junit4] 2> 5416684 INFO (zkCallback-2419-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5416692 INFO (zkCallback-2412-thread-1-processing-n:127.0.0.1:38119_dci%2Ffs) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5416720 INFO (zkCallback-2424-thread-2-processing-n:127.0.0.1:59392_dci%2Ffs) [n:127.0.0.1:59392_dci%2Ffs ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5417579 INFO (OverseerCollectionConfigSetProcessor-98639937054179332-127.0.0.1:38119_dci%2Ffs-n_0000000000) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5417950 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:59392_dci%2Ffs ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8
[junit4] 2> 5417981 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:59392_dci%2Ffs ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8
[junit4] 2> 5417981 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:59392_dci%2Ffs ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8
[junit4] 2> 5417982 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:59392_dci%2Ffs ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-1-001/cores
[junit4] 2> 5418056 INFO (qtp2108176362-54344) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:59392_dci%252Ffs&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5418069 INFO (OverseerThreadFactory-8020-thread-3-processing-n:127.0.0.1:38119_dci%2Ffs) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:59392_dci%2Ffs for creating new replica
[junit4] 2> 5418075 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 5418076 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 5418104 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 5418129 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema [collection1_shard2_replica_n0] Schema name=test
[junit4] 2> 5418651 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 5418679 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n0' using configuration from collection collection1, trusted=true
[junit4] 2> 5418680 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n0' (registry 'solr.core.collection1.shard2.replica_n0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8
[junit4] 2> 5418681 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5418681 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [[collection1_shard2_replica_n0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-1-001/cores/collection1_shard2_replica_n0], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-1-001/cores/collection1_shard2_replica_n0/data/]
[junit4] 2> 5418683 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=40, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.96484375, noCFSRatio=0.0]
[junit4] 2> 5418703 WARN (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 5418818 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5418818 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5418819 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 5418819 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5418832 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=27, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 5418838 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.SolrIndexSearcher Opening [Searcher@2e04af11[collection1_shard2_replica_n0] main]
[junit4] 2> 5418839 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 5418839 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 5418839 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 5418840 INFO (searcherExecutor-8034-thread-1-processing-n:127.0.0.1:59392_dci%2Ffs x:collection1_shard2_replica_n0 s:shard2 c:collection1) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [collection1_shard2_replica_n0] Registered new searcher Searcher@2e04af11[collection1_shard2_replica_n0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5418840 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578239002658471936
[junit4] 2> 5418846 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5418846 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 5418846 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:59392/dci/fs/collection1_shard2_replica_n0/
[junit4] 2> 5418846 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 5418846 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy http://127.0.0.1:59392/dci/fs/collection1_shard2_replica_n0/ has no replicas
[junit4] 2> 5418846 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 5418850 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:59392/dci/fs/collection1_shard2_replica_n0/ shard2
[junit4] 2> 5419001 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 5419002 INFO (qtp975467302-54387) [n:127.0.0.1:59392_dci%2Ffs c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=926
[junit4] 2> 5419004 INFO (qtp2108176362-54344) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:59392_dci%252Ffs&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=947
[junit4] 2> 5419006 INFO (OverseerCollectionConfigSetProcessor-98639937054179332-127.0.0.1:38119_dci%2Ffs-n_0000000000) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5419569 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-2-001 of type NRT
[junit4] 2> 5419570 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 5419643 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@645a8cc3{/dci/fs,null,AVAILABLE}
[junit4] 2> 5419643 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@59101533{HTTP/1.1,[http/1.1]}{127.0.0.1:51890}
[junit4] 2> 5419643 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.Server Started @5431787ms
[junit4] 2> 5419643 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/dci/fs, hostPort=51890, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-2-001/cores}
[junit4] 2> 5419643 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5419650 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 5419650 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5419650 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5419650 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-11T10:33:22.796Z
[junit4] 2> 5419674 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5419674 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-2-001/solr.xml
[junit4] 2> 5419678 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5419691 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57190/solr
[junit4] 2> 5419732 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:51890_dci%2Ffs ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 5419733 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:51890_dci%2Ffs ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5419735 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:51890_dci%2Ffs ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51890_dci%2Ffs
[junit4] 2> 5419735 INFO (zkCallback-2419-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5419742 INFO (zkCallback-2424-thread-2-processing-n:127.0.0.1:59392_dci%2Ffs) [n:127.0.0.1:59392_dci%2Ffs ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5419742 INFO (zkCallback-2412-thread-1-processing-n:127.0.0.1:38119_dci%2Ffs) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5419763 INFO (zkCallback-2430-thread-2-processing-n:127.0.0.1:51890_dci%2Ffs) [n:127.0.0.1:51890_dci%2Ffs ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5420583 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:51890_dci%2Ffs ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8
[junit4] 2> 5420603 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:51890_dci%2Ffs ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8
[junit4] 2> 5420603 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:51890_dci%2Ffs ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8
[junit4] 2> 5420604 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:51890_dci%2Ffs ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-2-001/cores
[junit4] 2> 5420694 INFO (qtp2108176362-54349) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:51890_dci%252Ffs&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5420701 INFO (OverseerThreadFactory-8020-thread-4-processing-n:127.0.0.1:38119_dci%2Ffs) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:51890_dci%2Ffs for creating new replica
[junit4] 2> 5420707 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 5420708 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 5421763 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 5421792 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.s.IndexSchema [collection1_shard1_replica_n0] Schema name=test
[junit4] 2> 5422011 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 5422030 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n0' using configuration from collection collection1, trusted=true
[junit4] 2> 5422031 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n0' (registry 'solr.core.collection1.shard1.replica_n0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8
[junit4] 2> 5422031 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5422031 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SolrCore [[collection1_shard1_replica_n0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-2-001/cores/collection1_shard1_replica_n0], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-2-001/cores/collection1_shard1_replica_n0/data/]
[junit4] 2> 5422034 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=40, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.96484375, noCFSRatio=0.0]
[junit4] 2> 5422056 WARN (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 5422253 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5422253 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5422253 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 5422254 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5422255 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=27, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 5422279 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.s.SolrIndexSearcher Opening [Searcher@2b22cb47[collection1_shard1_replica_n0] main]
[junit4] 2> 5422280 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 5422280 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 5422281 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 5422282 INFO (searcherExecutor-8045-thread-1-processing-n:127.0.0.1:51890_dci%2Ffs x:collection1_shard1_replica_n0 s:shard1 c:collection1) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SolrCore [collection1_shard1_replica_n0] Registered new searcher Searcher@2b22cb47[collection1_shard1_replica_n0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5422282 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1578239006267670528
[junit4] 2> 5422288 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5422288 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 5422288 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:51890/dci/fs/collection1_shard1_replica_n0/
[junit4] 2> 5422288 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 5422292 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SyncStrategy http://127.0.0.1:51890/dci/fs/collection1_shard1_replica_n0/ has no replicas
[junit4] 2> 5422292 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 5422295 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:51890/dci/fs/collection1_shard1_replica_n0/ shard1
[junit4] 2> 5422540 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 5422543 INFO (qtp1872278004-54417) [n:127.0.0.1:51890_dci%2Ffs c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1835
[junit4] 2> 5422545 INFO (qtp2108176362-54349) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:51890_dci%252Ffs&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1850
[junit4] 2> 5422701 INFO (OverseerCollectionConfigSetProcessor-98639937054179332-127.0.0.1:38119_dci%2Ffs-n_0000000000) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5422937 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-3-001 of type NRT
[junit4] 2> 5422938 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 5423008 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@c2853bc{/dci/fs,null,AVAILABLE}
[junit4] 2> 5423008 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5fbc19b6{HTTP/1.1,[http/1.1]}{127.0.0.1:60108}
[junit4] 2> 5423008 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.e.j.s.Server Started @5435152ms
[junit4] 2> 5423008 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/dci/fs, hostPort=60108, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-3-001/cores}
[junit4] 2> 5423009 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5423015 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 5423015 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5423015 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5423015 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-11T10:33:26.161Z
[junit4] 2> 5423056 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5423056 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-3-001/solr.xml
[junit4] 2> 5423062 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5423079 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57190/solr
[junit4] 2> 5423144 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:60108_dci%2Ffs ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 5423145 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:60108_dci%2Ffs ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5423147 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:60108_dci%2Ffs ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60108_dci%2Ffs
[junit4] 2> 5423147 INFO (zkCallback-2430-thread-2-processing-n:127.0.0.1:51890_dci%2Ffs) [n:127.0.0.1:51890_dci%2Ffs ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5423154 INFO (zkCallback-2419-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5423154 INFO (zkCallback-2424-thread-2-processing-n:127.0.0.1:59392_dci%2Ffs) [n:127.0.0.1:59392_dci%2Ffs ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5423154 INFO (zkCallback-2412-thread-1-processing-n:127.0.0.1:38119_dci%2Ffs) [n:127.0.0.1:38119_dci%2Ffs ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5423219 INFO (zkCallback-2436-thread-2-processing-n:127.0.0.1:60108_dci%2Ffs) [n:127.0.0.1:60108_dci%2Ffs ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5423704 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:60108_dci%2Ffs ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8
[junit4] 2> 5423883 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:60108_dci%2Ffs ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8
[junit4] 2> 5423883 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:60108_dci%2Ffs ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@45ace4f8
[junit4] 2> 5423884 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]) [n:127.0.0.1:60108_dci%2Ffs ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FB496A7D645EB255-001/shard-3-001/cores
[junit4] 2> 5424546 INFO (qtp1
[...truncated too long message...]
ark(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=54382, name=zkCallback-2419-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=54379, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=54843, name=zkCallback-2419-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 8) Thread[id=54381, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
[junit4] > at __randomizedtesting.SeedInfo.seed([FB496A7D645EB255]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=54844, name=zkCallback-2419-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 2) Thread[id=54898, name=zkCallback-2419-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 3) Thread[id=54380, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FB496A7D645EB255]-SendThread(127.0.0.1:57190), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > 4) Thread[id=54990, name=zkCallback-2419-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=54382, name=zkCallback-2419-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=54843, name=zkCallback-2419-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([FB496A7D645EB255]:0)
[junit4] Completed [459/729 (1!)] on J1 in 119.78s, 1 test, 1 failure, 2 errors <<< FAILURES!
[...truncated 44156 lines...]
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 41 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/41/
9 tests failed.
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([862E2454FBA12B3F:E7A1B8E555D46C7]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest: 1) Thread[id=64378, name=StoppableCommitThread-SendThread(127.0.0.1:32823), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 2) Thread[id=64112, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 3) Thread[id=64113, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55) 4) Thread[id=64495, name=zkCallback-1444-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=64497, name=zkCallback-1444-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=64496, name=zkCallback-1444-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=64380, name=zkCallback-1444-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=64111, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 9) Thread[id=64379, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:
1) Thread[id=64378, name=StoppableCommitThread-SendThread(127.0.0.1:32823), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
2) Thread[id=64112, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
3) Thread[id=64113, name=StoppableCommitThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableCommitThread.run(StoppableCommitThread.java:55)
4) Thread[id=64495, name=zkCallback-1444-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=64497, name=zkCallback-1444-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=64496, name=zkCallback-1444-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=64380, name=zkCallback-1444-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=64111, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
9) Thread[id=64379, name=StoppableCommitThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
at __randomizedtesting.SeedInfo.seed([862E2454FBA12B3F]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=64378, name=StoppableCommitThread-SendThread(127.0.0.1:32823), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 2) Thread[id=64112, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 3) Thread[id=64495, name=zkCallback-1444-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=64497, name=zkCallback-1444-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=64496, name=zkCallback-1444-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=64380, name=zkCallback-1444-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=64111, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at org.apache.solr.common.util.JavaBinCodec.writeStr(JavaBinCodec.java:841) at org.apache.solr.common.util.JavaBinCodec.writePrimitive(JavaBinCodec.java:932) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=64378, name=StoppableCommitThread-SendThread(127.0.0.1:32823), state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
2) Thread[id=64112, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
3) Thread[id=64495, name=zkCallback-1444-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=64497, name=zkCallback-1444-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=64496, name=zkCallback-1444-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=64380, name=zkCallback-1444-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=64111, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at org.apache.solr.common.util.JavaBinCodec.writeStr(JavaBinCodec.java:841)
at org.apache.solr.common.util.JavaBinCodec.writePrimitive(JavaBinCodec.java:932)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
at __randomizedtesting.SeedInfo.seed([862E2454FBA12B3F]:0)
FAILED: org.apache.solr.cloud.HttpPartitionTest.test
Error Message:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:60657/c8n_1x2_shard1_replica_n2
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:60657/c8n_1x2_shard1_replica_n2
at __randomizedtesting.SeedInfo.seed([862E2454FBA12B3F:E7A1B8E555D46C7]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:555)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:993)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:592)
at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:578)
at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:360)
at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:60657/c8n_1x2_shard1_replica_n2
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:638)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:549)
... 49 more
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)
... 54 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest
Error Message:
3 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest: 1) Thread[id=28856, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=28875, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=28864, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest:
1) Thread[id=28856, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=28875, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=28864, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([862E2454FBA12B3F]:0)
FAILED: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([862E2454FBA12B3F:E7A1B8E555D46C7]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([862E2454FBA12B3F:E7A1B8E555D46C7]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:174)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest.test
Error Message:
Error from server at http://127.0.0.1:57360/erz: ADDREPLICA failed to create replica
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:57360/erz: ADDREPLICA failed to create replica
at __randomizedtesting.SeedInfo.seed([862E2454FBA12B3F:E7A1B8E555D46C7]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.BasicDistributedZkTest.testANewCollectionInOneInstanceWithManualShardAssignement(BasicDistributedZkTest.java:801)
at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:374)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Timeout waiting for all live and active
Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
at __randomizedtesting.SeedInfo.seed([862E2454FBA12B3F:E7A1B8E555D46C7]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11677 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_862E2454FBA12B3F-001/init-core-data-001
[junit4] 2> 278778 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=14 numCloses=14
[junit4] 2> 278779 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 278780 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 278780 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 282945 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 285053 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 285477 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
[junit4] 2> 285712 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 286957 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 287145 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_35358_hdfs____.c3yle/webapp
[junit4] 2> 289128 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35358
[junit4] 2> 292911 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 292918 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 292954 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_45006_datanode____cu8jng/webapp
[junit4] 2> 293961 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45006
[junit4] 2> 295783 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 295785 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 295811 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_56836_datanode____.bxv3r3/webapp
[junit4] 2> 297255 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[862E2454FBA12B3F]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:56836
[junit4] 2> 299236 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_862E2454FBA12B3F-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_862E2454FBA12B3F-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:36061) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 299236 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_862E2454FBA12B3F-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_862E2454FBA12B3F-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:36061) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 300725 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x129b15205b33e6: from storage DS-a5f036c9-ac3a-4d7a-bae0-cc65888c3be7 node DatanodeRegistration(127.0.0.1:44267, datanodeUuid=43e12899-51e7-4566-a3c4-5207297eaf8e, infoPort=39495, infoSecurePort=0, ipcPort=40214, storageInfo=lv=-56;cid=testClusterID;nsid=1993716019;c=0), blocks: 0, hasStaleStorage: true, processing time: 3 msecs
[junit4] 2> 300725 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x129b151fc32661: from storage DS-0c652af4-3f9c-487c-af90-5716caeb4820 node DatanodeRegistration(127.0.0.1:58151, datanodeUuid=ca506d08-9b48-4be7-8df9-f3bf7aac6ec5, infoPort=52521, infoSecurePort=0, ipcPort=56541, storageInfo=lv=-56;cid=testClusterID;nsid=1993716019;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
[junit4] 2> 300725 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x129b15205b33e6: from storage DS-cb8d3287-1082-4ec4-8085-3910998ffc77 node DatanodeRegistration(127.0.0.1:44267, datanodeUuid=43e12899-51e7-4566-a3c4-5207297eaf8e, infoPort=39495, infoSecurePort=0, ipcPort=40214, storageInfo=lv=-56;cid=testClusterID;nsid=1993716019;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 300727 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x129b151fc32661: from storage DS-ad9eb8ee-e2d4-4053-8852-68145deef7e5 node DatanodeRegistration(127.0.0.1:58151, datanodeUuid=ca506d08-9b48-4be7-8df9-f3bf7aac6ec5, infoPort=52521, infoSecurePort=0, ipcPort=56541, storageInfo=lv=-56;cid=testClusterID;nsid=1993716019;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 302965 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 302968 INFO (Thread-435) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 302968 INFO (Thread-435) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 302992 ERROR (Thread-435) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 303068 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.ZkTestServer start zk server on port:53537
[junit4] 2> 303190 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 303192 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 303195 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 303196 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 303197 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 303197 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 303198 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 303199 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 303200 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 303201 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 303207 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 303212 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 303429 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 303430 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6634aaa6{/,null,AVAILABLE}
[junit4] 2> 303430 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@31449f52{HTTP/1.1,[http/1.1]}{127.0.0.1:37433}
[junit4] 2> 303430 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.e.j.s.Server Started @311485ms
[junit4] 2> 303431 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:36061/hdfs__localhost_36061__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.0_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_862E2454FBA12B3F-001_tempDir-002_control_data, hostContext=/, hostPort=37433, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_862E2454FBA12B3F-001/control-001/cores}
[junit4] 2> 303431 ERROR (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 303437 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 303437 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 303437 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 303437 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-09T19:25:39.954Z
[junit4] 2> 303480 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 303480 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_862E2454FBA12B3F-001/control-001/solr.xml
[junit4] 2> 303486 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@cb84273, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 303510 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53537/solr
[junit4] 2> 304244 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [n:127.0.0.1:37433_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 304249 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [n:127.0.0.1:37433_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37433_
[junit4] 2> 304254 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [n:127.0.0.1:37433_ ] o.a.s.c.Overseer Overseer (id=98630706102009860-127.0.0.1:37433_-n_0000000000) starting
[junit4] 2> 304331 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [n:127.0.0.1:37433_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37433_
[junit4] 2> 304332 INFO (zkCallback-223-thread-1-processing-n:127.0.0.1:37433_) [n:127.0.0.1:37433_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 304896 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [n:127.0.0.1:37433_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@cb84273
[junit4] 2> 304945 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [n:127.0.0.1:37433_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@cb84273
[junit4] 2> 304945 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [n:127.0.0.1:37433_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@cb84273
[junit4] 2> 304957 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [n:127.0.0.1:37433_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_862E2454FBA12B3F-001/control-001/cores
[junit4] 2> 305057 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 305057 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[862E2454FBA12B3F]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53537/solr ready
[junit4] 2> 305078 INFO (qtp453512589-1450) [n:127.0.0.1:37433_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37433_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 305095 INFO (OverseerThreadFactory-636-thread-1-processing-n:127.0.0.1:37433_) [n:127.0.0.1:37433_ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 305217 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 305219 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 305327 INFO (zkCallback-223-thread-1-processing-n:127.0.0.1:37433_) [n:127.0.0.1:37433_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 306272 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 306394 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 307410 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 307479 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 307480 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@cb84273
[junit4] 2> 308003 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:36061/solr_hdfs_home
[junit4] 2> 308004 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 308004 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 308004 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_862E2454FBA12B3F-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:36061/solr_hdfs_home/control_collection/core_node1/data/]
[junit4] 2> 308005 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:36061/solr_hdfs_home/control_collection/core_node1/data/snapshot_metadata
[junit4] 2> 308042 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 308042 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 308043 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new global HDFS BlockCache
[junit4] 2> 309615 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 309622 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:36061/solr_hdfs_home/control_collection/core_node1/data
[junit4] 2> 309752 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:36061/solr_hdfs_home/control_collection/core_node1/data/index
[junit4] 2> 309775 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 309775 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
[junit4] 2> 309794 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 309795 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=21, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 310785 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:58151 is added to blk_1073741825_1001{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-cb8d3287-1082-4ec4-8085-3910998ffc77:NORMAL:127.0.0.1:44267|RBW], ReplicaUC[[DISK]DS-ad9eb8ee-e2d4-4053-8852-68145deef7e5:NORMAL:127.0.0.1:58151|RBW]]} size 72
[junit4] 2> 310786 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:44267 is added to blk_1073741825_1001 size 72
[junit4] 2> 310984 WARN (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 311251 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 311251 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 311251 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 311378 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 311378 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 311403 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=49, maxMergedSegmentMB=1.98828125, floorSegmentMB=1.3603515625, forceMergeDeletesPctAllowed=29.131923403397916, segmentsPerTier=23.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4718482052455625
[junit4] 2> 312283 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@64212938[control_collection_shard1_replica_n1] main]
[junit4] 2> 312284 INFO (qtp453512589-1448) [n:127.0.0.1:37433_ c:control_colle
[...truncated too long message...]
eKnownType(JavaBinCodec.java:328)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
[junit4] > at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
[junit4] > at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
[junit4] > 3) Thread[id=64495, name=zkCallback-1444-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 4) Thread[id=64497, name=zkCallback-1444-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=64496, name=zkCallback-1444-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=64380, name=zkCallback-1444-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=64111, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeStr(JavaBinCodec.java:841)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writePrimitive(JavaBinCodec.java:932)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeSolrInputDocument(JavaBinCodec.java:616)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:355)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
[junit4] > at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
[junit4] > at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
[junit4] > at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
[junit4] > at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
[junit4] > at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
[junit4] > at __randomizedtesting.SeedInfo.seed([862E2454FBA12B3F]:0)
[junit4] Completed [297/729 (6!)] on J0 in 142.75s, 1 test, 1 failure, 2 errors <<< FAILURES!
[...truncated 44660 lines...]
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 40 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/40/
8 tests failed.
FAILED: junit.framework.TestSuite.org.apache.lucene.index.TestBackwardsCompatibility
Error Message:
Index name 6.6.1-cfs not found
Stack Trace:
java.lang.AssertionError: Index name 6.6.1-cfs not found
at __randomizedtesting.SeedInfo.seed([277DA2692EEAC86F]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNotNull(Assert.java:526)
at org.apache.lucene.index.TestBackwardsCompatibility.beforeClass(TestBackwardsCompatibility.java:470)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test
Error Message:
The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
Stack Trace:
java.lang.AssertionError: The Monkey ran for over 45 seconds and no jetties were stopped - this is worth investigating!
at __randomizedtesting.SeedInfo.seed([1DC5C60AE231867:898863BA00DF759F]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:587)
at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:222)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
Error Message:
11 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest: 1) Thread[id=18356, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 2) Thread[id=18151, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[1DC5C60AE231867]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 3) Thread[id=18149, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=18360, name=FullThrottleStopableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at org.apache.solr.cloud.FullThrottleStoppableIndexingThread.run(FullThrottleStoppableIndexingThread.java:90) 5) Thread[id=18150, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[1DC5C60AE231867]-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 6) Thread[id=18358, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135) 7) Thread[id=18359, name=StoppableSearchThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210) at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357) at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1126) at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1112) at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647) at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227) at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827) at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942) at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957) at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:54) 8) Thread[id=18471, name=StoppableSearchThread-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230) at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1246) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1170) 9) Thread[id=18472, name=StoppableSearchThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 10) Thread[id=18361, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=18357, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 11 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest:
1) Thread[id=18356, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
2) Thread[id=18151, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[1DC5C60AE231867]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
3) Thread[id=18149, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=18360, name=FullThrottleStopableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at org.apache.solr.cloud.FullThrottleStoppableIndexingThread.run(FullThrottleStoppableIndexingThread.java:90)
5) Thread[id=18150, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[1DC5C60AE231867]-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
6) Thread[id=18358, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
7) Thread[id=18359, name=StoppableSearchThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1126)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1112)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647)
at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1227)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:827)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:54)
8) Thread[id=18471, name=StoppableSearchThread-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230)
at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1246)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1170)
9) Thread[id=18472, name=StoppableSearchThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
10) Thread[id=18361, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=18357, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:135)
at __randomizedtesting.SeedInfo.seed([1DC5C60AE231867]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=18358, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Throwable.printStackTrace(Throwable.java:643) at java.lang.Throwable.printStackTrace(Throwable.java:634) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121) 2) Thread[id=18359, name=StoppableSearchThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at com.carrotsearch.ant.tasks.junit4.events.AbstractEvent.writeBinaryProperty(AbstractEvent.java:36) at com.carrotsearch.ant.tasks.junit4.events.AppendStdErrEvent.serialize(AppendStdErrEvent.java:30) at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:119) at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114) at java.security.AccessController.doPrivileged(Native Method) at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114) at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99) at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) at java.io.PrintStream.flush(PrintStream.java:338) at java.io.FilterOutputStream.flush(FilterOutputStream.java:140) at java.io.PrintStream.write(PrintStream.java:482) at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291) at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104) at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185) at java.io.PrintStream.write(PrintStream.java:527) at java.io.PrintStream.print(PrintStream.java:669) at java.io.PrintStream.println(PrintStream.java:823) at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748) at java.lang.Throwable.printEnclosedStackTrace(Throwable.java:697) at java.lang.Throwable.printStackTrace(Throwable.java:667) at java.lang.Throwable.printStackTrace(Throwable.java:643) at java.lang.Throwable.printStackTrace(Throwable.java:634) at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:57) 3) Thread[id=18356, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223) at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330) at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228) at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91) at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94) at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104) at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251) at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178) at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195) at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115) 4) Thread[id=18471, name=StoppableSearchThread-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 5) Thread[id=18357, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.io.PrintStream.println(PrintStream.java:805) at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120) 6) Thread[id=18360, name=FullThrottleStopableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.util.WeakHashMap.getTable(WeakHashMap.java:350) at java.util.WeakHashMap.getEntry(WeakHashMap.java:427) at java.util.WeakHashMap.containsKey(WeakHashMap.java:417) at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:258) at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134) at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:746) at org.apache.solr.cloud.FullThrottleStoppableIndexingThread.run(FullThrottleStoppableIndexingThread.java:76) 7) Thread[id=18150, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[1DC5C60AE231867]-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=18358, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Throwable.printStackTrace(Throwable.java:643)
at java.lang.Throwable.printStackTrace(Throwable.java:634)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:121)
2) Thread[id=18359, name=StoppableSearchThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at com.carrotsearch.ant.tasks.junit4.events.AbstractEvent.writeBinaryProperty(AbstractEvent.java:36)
at com.carrotsearch.ant.tasks.junit4.events.AppendStdErrEvent.serialize(AppendStdErrEvent.java:30)
at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:119)
at com.carrotsearch.ant.tasks.junit4.events.Serializer$2.run(Serializer.java:114)
at java.security.AccessController.doPrivileged(Native Method)
at com.carrotsearch.ant.tasks.junit4.events.Serializer.flushQueue(Serializer.java:114)
at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:99)
at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$3$2.write(SlaveMain.java:472)
at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
at java.io.PrintStream.flush(PrintStream.java:338)
at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
at java.io.PrintStream.write(PrintStream.java:482)
at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)
at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)
at java.io.PrintStream.write(PrintStream.java:527)
at java.io.PrintStream.print(PrintStream.java:669)
at java.io.PrintStream.println(PrintStream.java:823)
at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)
at java.lang.Throwable.printEnclosedStackTrace(Throwable.java:697)
at java.lang.Throwable.printStackTrace(Throwable.java:667)
at java.lang.Throwable.printStackTrace(Throwable.java:643)
at java.lang.Throwable.printStackTrace(Throwable.java:634)
at org.apache.solr.cloud.StoppableSearchThread.run(StoppableSearchThread.java:57)
3) Thread[id=18356, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:328)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeMapEntry(JavaBinCodec.java:764)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:383)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeIterator(JavaBinCodec.java:705)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:367)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.writeNamedList(JavaBinCodec.java:223)
at org.apache.solr.common.util.JavaBinCodec.writeKnownType(JavaBinCodec.java:330)
at org.apache.solr.common.util.JavaBinCodec.writeVal(JavaBinCodec.java:228)
at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:155)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.marshal(JavaBinUpdateRequestCodec.java:91)
at org.apache.solr.client.solrj.impl.BinaryRequestWriter.getContentStream(BinaryRequestWriter.java:67)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:94)
at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:104)
at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:382)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:176)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:115)
4) Thread[id=18471, name=StoppableSearchThread-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
5) Thread[id=18357, name=StoppableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.io.PrintStream.println(PrintStream.java:805)
at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:120)
6) Thread[id=18360, name=FullThrottleStopableIndexingThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.util.WeakHashMap.getTable(WeakHashMap.java:350)
at java.util.WeakHashMap.getEntry(WeakHashMap.java:427)
at java.util.WeakHashMap.containsKey(WeakHashMap.java:417)
at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:258)
at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:134)
at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:746)
at org.apache.solr.cloud.FullThrottleStoppableIndexingThread.run(FullThrottleStoppableIndexingThread.java:76)
7) Thread[id=18150, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[1DC5C60AE231867]-SendThread(127.0.0.1:48761), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
at __randomizedtesting.SeedInfo.seed([1DC5C60AE231867]:0)
FAILED: org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.moveReplicaTest
Error Message:
Error from server at http://127.0.0.1:47708/solr: ADDREPLICA failed to create replica
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:47708/solr: ADDREPLICA failed to create replica
at __randomizedtesting.SeedInfo.seed([1DC5C60AE231867:6FEE4FAF9B9A925]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.moveReplicaTest(HdfsCollectionsAPIDistributedZkTest.java:127)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload
Error Message:
expected:<[{indexVersion=1504863284898,generation=2,filelist=[_1dp.fdt, _1dp.fdx, _1dp.fnm, _1dp.nvd, _1dp.nvm, _1dp.si, _1dp_TestBloomFilteredLucenePostings_0.blm, _1dp_TestBloomFilteredLucenePostings_0.doc, _1dp_TestBloomFilteredLucenePostings_0.tim, _1dp_TestBloomFilteredLucenePostings_0.tip, _1eu.cfe, _1eu.cfs, _1eu.si, _1ev.cfe, _1ev.cfs, _1ev.si, _1ew.fdt, _1ew.fdx, _1ew.fnm, _1ew.nvd, _1ew.nvm, _1ew.si, _1ew_TestBloomFilteredLucenePostings_0.blm, _1ew_TestBloomFilteredLucenePostings_0.doc, _1ew_TestBloomFilteredLucenePostings_0.tim, _1ew_TestBloomFilteredLucenePostings_0.tip, _1ez.fdt, _1ez.fdx, _1ez.fnm, _1ez.nvd, _1ez.nvm, _1ez.si, _1ez_TestBloomFilteredLucenePostings_0.blm, _1ez_TestBloomFilteredLucenePostings_0.doc, _1ez_TestBloomFilteredLucenePostings_0.tim, _1ez_TestBloomFilteredLucenePostings_0.tip, _1f0.fdt, _1f0.fdx, _1f0.fnm, _1f0.nvd, _1f0.nvm, _1f0.si, _1f0_TestBloomFilteredLucenePostings_0.blm, _1f0_TestBloomFilteredLucenePostings_0.doc, _1f0_TestBloomFilteredLucenePostings_0.tim, _1f0_TestBloomFilteredLucenePostings_0.tip, _1f1.fdt, _1f1.fdx, _1f1.fnm, _1f1.nvd, _1f1.nvm, _1f1.si, _1f1_TestBloomFilteredLucenePostings_0.blm, _1f1_TestBloomFilteredLucenePostings_0.doc, _1f1_TestBloomFilteredLucenePostings_0.tim, _1f1_TestBloomFilteredLucenePostings_0.tip, _1f2.fdt, _1f2.fdx, _1f2.fnm, _1f2.nvd, _1f2.nvm, _1f2.si, _1f2_TestBloomFilteredLucenePostings_0.blm, _1f2_TestBloomFilteredLucenePostings_0.doc, _1f2_TestBloomFilteredLucenePostings_0.tim, _1f2_TestBloomFilteredLucenePostings_0.tip, _1f3.fdt, _1f3.fdx, _1f3.fnm, _1f3.nvd, _1f3.nvm, _1f3.si, _1f3_TestBloomFilteredLucenePostings_0.blm, _1f3_TestBloomFilteredLucenePostings_0.doc, _1f3_TestBloomFilteredLucenePostings_0.tim, _1f3_TestBloomFilteredLucenePostings_0.tip, _1f5.fdt, _1f5.fdx, _1f5.fnm, _1f5.nvd, _1f5.nvm, _1f5.si, _1f5_TestBloomFilteredLucenePostings_0.blm, _1f5_TestBloomFilteredLucenePostings_0.doc, _1f5_TestBloomFilteredLucenePostings_0.tim, _1f5_TestBloomFilteredLucenePostings_0.tip, _1f6.fdt, _1f6.fdx, _1f6.fnm, _1f6.nvd, _1f6.nvm, _1f6.si, _1f6_TestBloomFilteredLucenePostings_0.blm, _1f6_TestBloomFilteredLucenePostings_0.doc, _1f6_TestBloomFilteredLucenePostings_0.tim, _1f6_TestBloomFilteredLucenePostings_0.tip, _1fg.fdt, _1fg.fdx, _1fg.fnm, _1fg.nvd, _1fg.nvm, _1fg.si, _1fg_TestBloomFilteredLucenePostings_0.blm, _1fg_TestBloomFilteredLucenePostings_0.doc, _1fg_TestBloomFilteredLucenePostings_0.tim, _1fg_TestBloomFilteredLucenePostings_0.tip, _1fi.cfe, _1fi.cfs, _1fi.si, _1fk.cfe, _1fk.cfs, _1fk.si, _1fl.fdt, _1fl.fdx, _1fl.fnm, _1fl.nvd, _1fl.nvm, _1fl.si, _1fl_TestBloomFilteredLucenePostings_0.blm, _1fl_TestBloomFilteredLucenePostings_0.doc, _1fl_TestBloomFilteredLucenePostings_0.tim, _1fl_TestBloomFilteredLucenePostings_0.tip, _1fm.fdt, _1fm.fdx, _1fm.fnm, _1fm.nvd, _1fm.nvm, _1fm.si, _1fm_TestBloomFilteredLucenePostings_0.blm, _1fm_TestBloomFilteredLucenePostings_0.doc, _1fm_TestBloomFilteredLucenePostings_0.tim, _1fm_TestBloomFilteredLucenePostings_0.tip, _1fn.fdt, _1fn.fdx, _1fn.fnm, _1fn.nvd, _1fn.nvm, _1fn.si, _1fn_TestBloomFilteredLucenePostings_0.blm, _1fn_TestBloomFilteredLucenePostings_0.doc, _1fn_TestBloomFilteredLucenePostings_0.tim, _1fn_TestBloomFilteredLucenePostings_0.tip, _1fo.fdt, _1fo.fdx, _1fo.fnm, _1fo.nvd, _1fo.nvm, _1fo.si, _1fo_TestBloomFilteredLucenePostings_0.blm, _1fo_TestBloomFilteredLucenePostings_0.doc, _1fo_TestBloomFilteredLucenePostings_0.tim, _1fo_TestBloomFilteredLucenePostings_0.tip, segments_2]}]> but was:<[{indexVersion=1504863284898,generation=2,filelist=[_1dp.fdt, _1dp.fdx, _1dp.fnm, _1dp.nvd, _1dp.nvm, _1dp.si, _1dp_TestBloomFilteredLucenePostings_0.blm, _1dp_TestBloomFilteredLucenePostings_0.doc, _1dp_TestBloomFilteredLucenePostings_0.tim, _1dp_TestBloomFilteredLucenePostings_0.tip, _1eu.cfe, _1eu.cfs, _1eu.si, _1ev.cfe, _1ev.cfs, _1ev.si, _1ew.fdt, _1ew.fdx, _1ew.fnm, _1ew.nvd, _1ew.nvm, _1ew.si, _1ew_TestBloomFilteredLucenePostings_0.blm, _1ew_TestBloomFilteredLucenePostings_0.doc, _1ew_TestBloomFilteredLucenePostings_0.tim, _1ew_TestBloomFilteredLucenePostings_0.tip, _1ez.fdt, _1ez.fdx, _1ez.fnm, _1ez.nvd, _1ez.nvm, _1ez.si, _1ez_TestBloomFilteredLucenePostings_0.blm, _1ez_TestBloomFilteredLucenePostings_0.doc, _1ez_TestBloomFilteredLucenePostings_0.tim, _1ez_TestBloomFilteredLucenePostings_0.tip, _1f0.fdt, _1f0.fdx, _1f0.fnm, _1f0.nvd, _1f0.nvm, _1f0.si, _1f0_TestBloomFilteredLucenePostings_0.blm, _1f0_TestBloomFilteredLucenePostings_0.doc, _1f0_TestBloomFilteredLucenePostings_0.tim, _1f0_TestBloomFilteredLucenePostings_0.tip, _1f1.fdt, _1f1.fdx, _1f1.fnm, _1f1.nvd, _1f1.nvm, _1f1.si, _1f1_TestBloomFilteredLucenePostings_0.blm, _1f1_TestBloomFilteredLucenePostings_0.doc, _1f1_TestBloomFilteredLucenePostings_0.tim, _1f1_TestBloomFilteredLucenePostings_0.tip, _1f2.fdt, _1f2.fdx, _1f2.fnm, _1f2.nvd, _1f2.nvm, _1f2.si, _1f2_TestBloomFilteredLucenePostings_0.blm, _1f2_TestBloomFilteredLucenePostings_0.doc, _1f2_TestBloomFilteredLucenePostings_0.tim, _1f2_TestBloomFilteredLucenePostings_0.tip, _1f3.fdt, _1f3.fdx, _1f3.fnm, _1f3.nvd, _1f3.nvm, _1f3.si, _1f3_TestBloomFilteredLucenePostings_0.blm, _1f3_TestBloomFilteredLucenePostings_0.doc, _1f3_TestBloomFilteredLucenePostings_0.tim, _1f3_TestBloomFilteredLucenePostings_0.tip, _1f5.fdt, _1f5.fdx, _1f5.fnm, _1f5.nvd, _1f5.nvm, _1f5.si, _1f5_TestBloomFilteredLucenePostings_0.blm, _1f5_TestBloomFilteredLucenePostings_0.doc, _1f5_TestBloomFilteredLucenePostings_0.tim, _1f5_TestBloomFilteredLucenePostings_0.tip, _1f6.fdt, _1f6.fdx, _1f6.fnm, _1f6.nvd, _1f6.nvm, _1f6.si, _1f6_TestBloomFilteredLucenePostings_0.blm, _1f6_TestBloomFilteredLucenePostings_0.doc, _1f6_TestBloomFilteredLucenePostings_0.tim, _1f6_TestBloomFilteredLucenePostings_0.tip, _1fg.fdt, _1fg.fdx, _1fg.fnm, _1fg.nvd, _1fg.nvm, _1fg.si, _1fg_TestBloomFilteredLucenePostings_0.blm, _1fg_TestBloomFilteredLucenePostings_0.doc, _1fg_TestBloomFilteredLucenePostings_0.tim, _1fg_TestBloomFilteredLucenePostings_0.tip, _1fi.cfe, _1fi.cfs, _1fi.si, _1fk.cfe, _1fk.cfs, _1fk.si, _1fl.fdt, _1fl.fdx, _1fl.fnm, _1fl.nvd, _1fl.nvm, _1fl.si, _1fl_TestBloomFilteredLucenePostings_0.blm, _1fl_TestBloomFilteredLucenePostings_0.doc, _1fl_TestBloomFilteredLucenePostings_0.tim, _1fl_TestBloomFilteredLucenePostings_0.tip, _1fm.fdt, _1fm.fdx, _1fm.fnm, _1fm.nvd, _1fm.nvm, _1fm.si, _1fm_TestBloomFilteredLucenePostings_0.blm, _1fm_TestBloomFilteredLucenePostings_0.doc, _1fm_TestBloomFilteredLucenePostings_0.tim, _1fm_TestBloomFilteredLucenePostings_0.tip, _1fn.fdt, _1fn.fdx, _1fn.fnm, _1fn.nvd, _1fn.nvm, _1fn.si, _1fn_TestBloomFilteredLucenePostings_0.blm, _1fn_TestBloomFilteredLucenePostings_0.doc, _1fn_TestBloomFilteredLucenePostings_0.tim, _1fn_TestBloomFilteredLucenePostings_0.tip, _1fo.fdt, _1fo.fdx, _1fo.fnm, _1fo.nvd, _1fo.nvm, _1fo.si, _1fo_TestBloomFilteredLucenePostings_0.blm, _1fo_TestBloomFilteredLucenePostings_0.doc, _1fo_TestBloomFilteredLucenePostings_0.tim, _1fo_TestBloomFilteredLucenePostings_0.tip, segments_2]}, {indexVersion=1504863284898,generation=3,filelist=[_1fp.cfe, _1fp.cfs, _1fp.si, segments_3]}]>
Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1504863284898,generation=2,filelist=[_1dp.fdt, _1dp.fdx, _1dp.fnm, _1dp.nvd, _1dp.nvm, _1dp.si, _1dp_TestBloomFilteredLucenePostings_0.blm, _1dp_TestBloomFilteredLucenePostings_0.doc, _1dp_TestBloomFilteredLucenePostings_0.tim, _1dp_TestBloomFilteredLucenePostings_0.tip, _1eu.cfe, _1eu.cfs, _1eu.si, _1ev.cfe, _1ev.cfs, _1ev.si, _1ew.fdt, _1ew.fdx, _1ew.fnm, _1ew.nvd, _1ew.nvm, _1ew.si, _1ew_TestBloomFilteredLucenePostings_0.blm, _1ew_TestBloomFilteredLucenePostings_0.doc, _1ew_TestBloomFilteredLucenePostings_0.tim, _1ew_TestBloomFilteredLucenePostings_0.tip, _1ez.fdt, _1ez.fdx, _1ez.fnm, _1ez.nvd, _1ez.nvm, _1ez.si, _1ez_TestBloomFilteredLucenePostings_0.blm, _1ez_TestBloomFilteredLucenePostings_0.doc, _1ez_TestBloomFilteredLucenePostings_0.tim, _1ez_TestBloomFilteredLucenePostings_0.tip, _1f0.fdt, _1f0.fdx, _1f0.fnm, _1f0.nvd, _1f0.nvm, _1f0.si, _1f0_TestBloomFilteredLucenePostings_0.blm, _1f0_TestBloomFilteredLucenePostings_0.doc, _1f0_TestBloomFilteredLucenePostings_0.tim, _1f0_TestBloomFilteredLucenePostings_0.tip, _1f1.fdt, _1f1.fdx, _1f1.fnm, _1f1.nvd, _1f1.nvm, _1f1.si, _1f1_TestBloomFilteredLucenePostings_0.blm, _1f1_TestBloomFilteredLucenePostings_0.doc, _1f1_TestBloomFilteredLucenePostings_0.tim, _1f1_TestBloomFilteredLucenePostings_0.tip, _1f2.fdt, _1f2.fdx, _1f2.fnm, _1f2.nvd, _1f2.nvm, _1f2.si, _1f2_TestBloomFilteredLucenePostings_0.blm, _1f2_TestBloomFilteredLucenePostings_0.doc, _1f2_TestBloomFilteredLucenePostings_0.tim, _1f2_TestBloomFilteredLucenePostings_0.tip, _1f3.fdt, _1f3.fdx, _1f3.fnm, _1f3.nvd, _1f3.nvm, _1f3.si, _1f3_TestBloomFilteredLucenePostings_0.blm, _1f3_TestBloomFilteredLucenePostings_0.doc, _1f3_TestBloomFilteredLucenePostings_0.tim, _1f3_TestBloomFilteredLucenePostings_0.tip, _1f5.fdt, _1f5.fdx, _1f5.fnm, _1f5.nvd, _1f5.nvm, _1f5.si, _1f5_TestBloomFilteredLucenePostings_0.blm, _1f5_TestBloomFilteredLucenePostings_0.doc, _1f5_TestBloomFilteredLucenePostings_0.tim, _1f5_TestBloomFilteredLucenePostings_0.tip, _1f6.fdt, _1f6.fdx, _1f6.fnm, _1f6.nvd, _1f6.nvm, _1f6.si, _1f6_TestBloomFilteredLucenePostings_0.blm, _1f6_TestBloomFilteredLucenePostings_0.doc, _1f6_TestBloomFilteredLucenePostings_0.tim, _1f6_TestBloomFilteredLucenePostings_0.tip, _1fg.fdt, _1fg.fdx, _1fg.fnm, _1fg.nvd, _1fg.nvm, _1fg.si, _1fg_TestBloomFilteredLucenePostings_0.blm, _1fg_TestBloomFilteredLucenePostings_0.doc, _1fg_TestBloomFilteredLucenePostings_0.tim, _1fg_TestBloomFilteredLucenePostings_0.tip, _1fi.cfe, _1fi.cfs, _1fi.si, _1fk.cfe, _1fk.cfs, _1fk.si, _1fl.fdt, _1fl.fdx, _1fl.fnm, _1fl.nvd, _1fl.nvm, _1fl.si, _1fl_TestBloomFilteredLucenePostings_0.blm, _1fl_TestBloomFilteredLucenePostings_0.doc, _1fl_TestBloomFilteredLucenePostings_0.tim, _1fl_TestBloomFilteredLucenePostings_0.tip, _1fm.fdt, _1fm.fdx, _1fm.fnm, _1fm.nvd, _1fm.nvm, _1fm.si, _1fm_TestBloomFilteredLucenePostings_0.blm, _1fm_TestBloomFilteredLucenePostings_0.doc, _1fm_TestBloomFilteredLucenePostings_0.tim, _1fm_TestBloomFilteredLucenePostings_0.tip, _1fn.fdt, _1fn.fdx, _1fn.fnm, _1fn.nvd, _1fn.nvm, _1fn.si, _1fn_TestBloomFilteredLucenePostings_0.blm, _1fn_TestBloomFilteredLucenePostings_0.doc, _1fn_TestBloomFilteredLucenePostings_0.tim, _1fn_TestBloomFilteredLucenePostings_0.tip, _1fo.fdt, _1fo.fdx, _1fo.fnm, _1fo.nvd, _1fo.nvm, _1fo.si, _1fo_TestBloomFilteredLucenePostings_0.blm, _1fo_TestBloomFilteredLucenePostings_0.doc, _1fo_TestBloomFilteredLucenePostings_0.tim, _1fo_TestBloomFilteredLucenePostings_0.tip, segments_2]}]> but was:<[{indexVersion=1504863284898,generation=2,filelist=[_1dp.fdt, _1dp.fdx, _1dp.fnm, _1dp.nvd, _1dp.nvm, _1dp.si, _1dp_TestBloomFilteredLucenePostings_0.blm, _1dp_TestBloomFilteredLucenePostings_0.doc, _1dp_TestBloomFilteredLucenePostings_0.tim, _1dp_TestBloomFilteredLucenePostings_0.tip, _1eu.cfe, _1eu.cfs, _1eu.si, _1ev.cfe, _1ev.cfs, _1ev.si, _1ew.fdt, _1ew.fdx, _1ew.fnm, _1ew.nvd, _1ew.nvm, _1ew.si, _1ew_TestBloomFilteredLucenePostings_0.blm, _1ew_TestBloomFilteredLucenePostings_0.doc, _1ew_TestBloomFilteredLucenePostings_0.tim, _1ew_TestBloomFilteredLucenePostings_0.tip, _1ez.fdt, _1ez.fdx, _1ez.fnm, _1ez.nvd, _1ez.nvm, _1ez.si, _1ez_TestBloomFilteredLucenePostings_0.blm, _1ez_TestBloomFilteredLucenePostings_0.doc, _1ez_TestBloomFilteredLucenePostings_0.tim, _1ez_TestBloomFilteredLucenePostings_0.tip, _1f0.fdt, _1f0.fdx, _1f0.fnm, _1f0.nvd, _1f0.nvm, _1f0.si, _1f0_TestBloomFilteredLucenePostings_0.blm, _1f0_TestBloomFilteredLucenePostings_0.doc, _1f0_TestBloomFilteredLucenePostings_0.tim, _1f0_TestBloomFilteredLucenePostings_0.tip, _1f1.fdt, _1f1.fdx, _1f1.fnm, _1f1.nvd, _1f1.nvm, _1f1.si, _1f1_TestBloomFilteredLucenePostings_0.blm, _1f1_TestBloomFilteredLucenePostings_0.doc, _1f1_TestBloomFilteredLucenePostings_0.tim, _1f1_TestBloomFilteredLucenePostings_0.tip, _1f2.fdt, _1f2.fdx, _1f2.fnm, _1f2.nvd, _1f2.nvm, _1f2.si, _1f2_TestBloomFilteredLucenePostings_0.blm, _1f2_TestBloomFilteredLucenePostings_0.doc, _1f2_TestBloomFilteredLucenePostings_0.tim, _1f2_TestBloomFilteredLucenePostings_0.tip, _1f3.fdt, _1f3.fdx, _1f3.fnm, _1f3.nvd, _1f3.nvm, _1f3.si, _1f3_TestBloomFilteredLucenePostings_0.blm, _1f3_TestBloomFilteredLucenePostings_0.doc, _1f3_TestBloomFilteredLucenePostings_0.tim, _1f3_TestBloomFilteredLucenePostings_0.tip, _1f5.fdt, _1f5.fdx, _1f5.fnm, _1f5.nvd, _1f5.nvm, _1f5.si, _1f5_TestBloomFilteredLucenePostings_0.blm, _1f5_TestBloomFilteredLucenePostings_0.doc, _1f5_TestBloomFilteredLucenePostings_0.tim, _1f5_TestBloomFilteredLucenePostings_0.tip, _1f6.fdt, _1f6.fdx, _1f6.fnm, _1f6.nvd, _1f6.nvm, _1f6.si, _1f6_TestBloomFilteredLucenePostings_0.blm, _1f6_TestBloomFilteredLucenePostings_0.doc, _1f6_TestBloomFilteredLucenePostings_0.tim, _1f6_TestBloomFilteredLucenePostings_0.tip, _1fg.fdt, _1fg.fdx, _1fg.fnm, _1fg.nvd, _1fg.nvm, _1fg.si, _1fg_TestBloomFilteredLucenePostings_0.blm, _1fg_TestBloomFilteredLucenePostings_0.doc, _1fg_TestBloomFilteredLucenePostings_0.tim, _1fg_TestBloomFilteredLucenePostings_0.tip, _1fi.cfe, _1fi.cfs, _1fi.si, _1fk.cfe, _1fk.cfs, _1fk.si, _1fl.fdt, _1fl.fdx, _1fl.fnm, _1fl.nvd, _1fl.nvm, _1fl.si, _1fl_TestBloomFilteredLucenePostings_0.blm, _1fl_TestBloomFilteredLucenePostings_0.doc, _1fl_TestBloomFilteredLucenePostings_0.tim, _1fl_TestBloomFilteredLucenePostings_0.tip, _1fm.fdt, _1fm.fdx, _1fm.fnm, _1fm.nvd, _1fm.nvm, _1fm.si, _1fm_TestBloomFilteredLucenePostings_0.blm, _1fm_TestBloomFilteredLucenePostings_0.doc, _1fm_TestBloomFilteredLucenePostings_0.tim, _1fm_TestBloomFilteredLucenePostings_0.tip, _1fn.fdt, _1fn.fdx, _1fn.fnm, _1fn.nvd, _1fn.nvm, _1fn.si, _1fn_TestBloomFilteredLucenePostings_0.blm, _1fn_TestBloomFilteredLucenePostings_0.doc, _1fn_TestBloomFilteredLucenePostings_0.tim, _1fn_TestBloomFilteredLucenePostings_0.tip, _1fo.fdt, _1fo.fdx, _1fo.fnm, _1fo.nvd, _1fo.nvm, _1fo.si, _1fo_TestBloomFilteredLucenePostings_0.blm, _1fo_TestBloomFilteredLucenePostings_0.doc, _1fo_TestBloomFilteredLucenePostings_0.tim, _1fo_TestBloomFilteredLucenePostings_0.tip, segments_2]}, {indexVersion=1504863284898,generation=3,filelist=[_1fp.cfe, _1fp.cfs, _1fp.si, segments_3]}]>
at __randomizedtesting.SeedInfo.seed([1DC5C60AE231867:240B4750DE6B1664]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1277)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.logging.TestLogWatcher.testLog4jWatcher
Error Message:
expected:<3> but was:<1>
Stack Trace:
java.lang.AssertionError: expected:<3> but was:<1>
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.logging.TestLogWatcher.testLog4jWatcher(TestLogWatcher.java:52)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:263)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:68)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:47)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:60)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:229)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:50)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:222)
at org.junit.runners.ParentRunner.run(ParentRunner.java:300)
at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
expected:<2> but was:<3>
Stack Trace:
java.lang.AssertionError: expected:<2> but was:<3>
at __randomizedtesting.SeedInfo.seed([1DC5C60AE231867:898863BA00DF759F]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:403)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:248)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 5063 lines...]
[junit4] Suite: org.apache.lucene.index.TestBackwardsCompatibility
[junit4] 2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=false): {}, locale=mt, timezone=Europe/Stockholm
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=181440968,total=342360064
[junit4] 2> NOTE: All tests run in this JVM: [TestManyPointsInOldIndex, TestLucene50SegmentInfoFormat, TestBackwardsCompatibility]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestBackwardsCompatibility -Dtests.seed=277DA2692EEAC86F -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=mt -Dtests.timezone=Europe/Stockholm -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | TestBackwardsCompatibility (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: Index name 6.6.1-cfs not found
[junit4] > at __randomizedtesting.SeedInfo.seed([277DA2692EEAC86F]:0)
[junit4] > at org.apache.lucene.index.TestBackwardsCompatibility.beforeClass(TestBackwardsCompatibility.java:470)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] Completed [3/8 (1!)] on J0 in 4.87s, 0 tests, 1 failure <<< FAILURES!
[...truncated 7180 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1DC5C60AE231867-001/init-core-data-001
[junit4] 2> 1851563 WARN (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=16 numCloses=16
[junit4] 2> 1851563 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 1851569 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 1851569 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1DC5C60AE231867-001/tempDir-001
[junit4] 2> 1851570 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1851571 INFO (Thread-9272) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1851571 INFO (Thread-9272) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1851581 ERROR (Thread-9272) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1851671 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:39869
[junit4] 2> 1851697 INFO (jetty-launcher-1350-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1851698 INFO (jetty-launcher-1350-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1851704 INFO (jetty-launcher-1350-thread-3) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1851706 INFO (jetty-launcher-1350-thread-4) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1851724 INFO (jetty-launcher-1350-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5ef9e63{/solr,null,AVAILABLE}
[junit4] 2> 1851725 INFO (jetty-launcher-1350-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5986f472{HTTP/1.1,[http/1.1]}{127.0.0.1:56783}
[junit4] 2> 1851725 INFO (jetty-launcher-1350-thread-2) [ ] o.e.j.s.Server Started @1891176ms
[junit4] 2> 1851725 INFO (jetty-launcher-1350-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=56783}
[junit4] 2> 1851725 ERROR (jetty-launcher-1350-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1851725 INFO (jetty-launcher-1350-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1851726 INFO (jetty-launcher-1350-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1851726 INFO (jetty-launcher-1350-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1851726 INFO (jetty-launcher-1350-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-08T09:30:52.630Z
[junit4] 2> 1851741 INFO (jetty-launcher-1350-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@360d2e1{/solr,null,AVAILABLE}
[junit4] 2> 1851741 INFO (jetty-launcher-1350-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2e4a4c0c{HTTP/1.1,[http/1.1]}{127.0.0.1:35324}
[junit4] 2> 1851741 INFO (jetty-launcher-1350-thread-1) [ ] o.e.j.s.Server Started @1891192ms
[junit4] 2> 1851741 INFO (jetty-launcher-1350-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35324}
[junit4] 2> 1851741 ERROR (jetty-launcher-1350-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1851741 INFO (jetty-launcher-1350-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1851741 INFO (jetty-launcher-1350-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1851741 INFO (jetty-launcher-1350-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1851741 INFO (jetty-launcher-1350-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-08T09:30:52.645Z
[junit4] 2> 1851749 INFO (jetty-launcher-1350-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1851756 INFO (jetty-launcher-1350-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@47fce118{/solr,null,AVAILABLE}
[junit4] 2> 1851757 INFO (jetty-launcher-1350-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@12a0469f{HTTP/1.1,[http/1.1]}{127.0.0.1:47708}
[junit4] 2> 1851757 INFO (jetty-launcher-1350-thread-4) [ ] o.e.j.s.Server Started @1891208ms
[junit4] 2> 1851757 INFO (jetty-launcher-1350-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=47708}
[junit4] 2> 1851757 ERROR (jetty-launcher-1350-thread-4) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1851757 INFO (jetty-launcher-1350-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1851757 INFO (jetty-launcher-1350-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1851757 INFO (jetty-launcher-1350-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1851757 INFO (jetty-launcher-1350-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-08T09:30:52.661Z
[junit4] 2> 1851759 INFO (jetty-launcher-1350-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@79b9fa25{/solr,null,AVAILABLE}
[junit4] 2> 1851759 INFO (jetty-launcher-1350-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@37fe371{HTTP/1.1,[http/1.1]}{127.0.0.1:52792}
[junit4] 2> 1851759 INFO (jetty-launcher-1350-thread-3) [ ] o.e.j.s.Server Started @1891210ms
[junit4] 2> 1851759 INFO (jetty-launcher-1350-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52792}
[junit4] 2> 1851760 ERROR (jetty-launcher-1350-thread-3) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1851760 INFO (jetty-launcher-1350-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1851760 INFO (jetty-launcher-1350-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1851760 INFO (jetty-launcher-1350-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1851760 INFO (jetty-launcher-1350-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-08T09:30:52.664Z
[junit4] 2> 1851765 INFO (jetty-launcher-1350-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1851779 INFO (jetty-launcher-1350-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39869/solr
[junit4] 2> 1851791 INFO (jetty-launcher-1350-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1851808 INFO (jetty-launcher-1350-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39869/solr
[junit4] 2> 1851818 INFO (jetty-launcher-1350-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1851864 INFO (jetty-launcher-1350-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39869/solr
[junit4] 2> 1851911 INFO (jetty-launcher-1350-thread-2) [n:127.0.0.1:56783_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1851931 INFO (jetty-launcher-1350-thread-1) [n:127.0.0.1:35324_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1851939 INFO (jetty-launcher-1350-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39869/solr
[junit4] 2> 1851940 INFO (jetty-launcher-1350-thread-2) [n:127.0.0.1:56783_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56783_solr
[junit4] 2> 1851941 INFO (jetty-launcher-1350-thread-1) [n:127.0.0.1:35324_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35324_solr
[junit4] 2> 1851952 INFO (jetty-launcher-1350-thread-2) [n:127.0.0.1:56783_solr ] o.a.s.c.Overseer Overseer (id=98622705007067144-127.0.0.1:56783_solr-n_0000000000) starting
[junit4] 2> 1851985 INFO (jetty-launcher-1350-thread-2) [n:127.0.0.1:56783_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56783_solr
[junit4] 2> 1852204 INFO (zkCallback-1365-thread-1-processing-n:127.0.0.1:56783_solr) [n:127.0.0.1:56783_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1852204 INFO (zkCallback-1364-thread-1-processing-n:127.0.0.1:35324_solr) [n:127.0.0.1:35324_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1852213 INFO (zkCallback-1364-thread-2-processing-n:127.0.0.1:35324_solr) [n:127.0.0.1:35324_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1852257 INFO (jetty-launcher-1350-thread-4) [n:127.0.0.1:47708_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1852258 INFO (jetty-launcher-1350-thread-4) [n:127.0.0.1:47708_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1852259 INFO (jetty-launcher-1350-thread-4) [n:127.0.0.1:47708_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47708_solr
[junit4] 2> 1852260 INFO (zkCallback-1365-thread-1-processing-n:127.0.0.1:56783_solr) [n:127.0.0.1:56783_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1852262 INFO (zkCallback-1364-thread-2-processing-n:127.0.0.1:35324_solr) [n:127.0.0.1:35324_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1852340 INFO (zkCallback-1373-thread-1-processing-n:127.0.0.1:47708_solr) [n:127.0.0.1:47708_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1852346 INFO (jetty-launcher-1350-thread-3) [n:127.0.0.1:52792_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1852347 INFO (jetty-launcher-1350-thread-3) [n:127.0.0.1:52792_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1852348 INFO (jetty-launcher-1350-thread-3) [n:127.0.0.1:52792_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52792_solr
[junit4] 2> 1852349 INFO (zkCallback-1365-thread-1-processing-n:127.0.0.1:56783_solr) [n:127.0.0.1:56783_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1852349 INFO (zkCallback-1364-thread-2-processing-n:127.0.0.1:35324_solr) [n:127.0.0.1:35324_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1852349 INFO (zkCallback-1373-thread-1-processing-n:127.0.0.1:47708_solr) [n:127.0.0.1:47708_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1852373 INFO (zkCallback-1374-thread-1-processing-n:127.0.0.1:52792_solr) [n:127.0.0.1:52792_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1852446 INFO (jetty-launcher-1350-thread-2) [n:127.0.0.1:56783_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49075ae
[junit4] 2> 1852486 INFO (jetty-launcher-1350-thread-2) [n:127.0.0.1:56783_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49075ae
[junit4] 2> 1852486 INFO (jetty-launcher-1350-thread-2) [n:127.0.0.1:56783_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49075ae
[junit4] 2> 1852487 INFO (jetty-launcher-1350-thread-2) [n:127.0.0.1:56783_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1DC5C60AE231867-001/tempDir-001/node2
[junit4] 2> 1852520 INFO (jetty-launcher-1350-thread-3) [n:127.0.0.1:52792_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49075ae
[junit4] 2> 1852522 INFO (jetty-launcher-1350-thread-4) [n:127.0.0.1:47708_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49075ae
[junit4] 2> 1852584 INFO (jetty-launcher-1350-thread-3) [n:127.0.0.1:52792_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49075ae
[junit4] 2> 1852586 INFO (jetty-launcher-1350-thread-1) [n:127.0.0.1:35324_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49075ae
[junit4] 2> 1852596 INFO (jetty-launcher-1350-thread-3) [n:127.0.0.1:52792_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49075ae
[junit4] 2> 1852597 INFO (jetty-launcher-1350-thread-3) [n:127.0.0.1:52792_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1DC5C60AE231867-001/tempDir-001/node3
[junit4] 2> 1852602 INFO (jetty-launcher-1350-thread-4) [n:127.0.0.1:47708_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49075ae
[junit4] 2> 1852602 INFO (jetty-launcher-1350-thread-4) [n:127.0.0.1:47708_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49075ae
[junit4] 2> 1852603 INFO (jetty-launcher-1350-thread-4) [n:127.0.0.1:47708_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1DC5C60AE231867-001/tempDir-001/node4
[junit4] 2> 1852631 INFO (jetty-launcher-1350-thread-1) [n:127.0.0.1:35324_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49075ae
[junit4] 2> 1852631 INFO (jetty-launcher-1350-thread-1) [n:127.0.0.1:35324_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@49075ae
[junit4] 2> 1852634 INFO (jetty-launcher-1350-thread-1) [n:127.0.0.1:35324_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1DC5C60AE231867-001/tempDir-001/node1
[junit4] 2> 1853091 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1853092 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39869/solr ready
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 1853151 WARN (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 1853156 WARN (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1853159 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1853188 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_42310_hdfs____rg775q/webapp
[junit4] 2> 1853734 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42310
[junit4] 2> 1853858 WARN (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1853864 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1853888 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_55827_datanode____en9g/webapp
[junit4] 2> 1854367 INFO (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1DC5C60AE231867]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:55827
[junit4] 2> 1854586 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1DC5C60AE231867-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1DC5C60AE231867-001/tempDir-002/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:43344) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 1854614 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x122c0cc3319c97: from storage DS-87c7c162-9236-4b91-a11b-76dc8a266a9b node DatanodeRegistration(127.0.0.1:44656, datanodeUuid=af952740-62bf-4238-a222-28b923e19fce, infoPort=56395, infoSecurePort=0, ipcPort=35539, storageInfo=lv=-56;cid=testClusterID;nsid=1499042934;c=0), blocks: 0, hasStaleStorage: true, processing time: 6 msecs
[junit4] 2> 1854614 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x122c0cc3319c97: from storage DS-4ff55d10-a18f-4c57-b6a6-41799621e9b8 node DatanodeRegistration(127.0.0.1:44656, datanodeUuid=af952740-62bf-4238-a222-28b923e19fce, infoPort=56395, infoSecurePort=0, ipcPort=35539, storageInfo=lv=-56;cid=testClusterID;nsid=1499042934;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 1854797 INFO (TEST-HdfsCollectionsAPIDistributedZkTest.testCreateNodeSet-seed#[1DC5C60AE231867]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testCreateNodeSet
[junit4] 2> 1854798 INFO (TEST-HdfsCollectionsAPIDistributedZkTest.testCreateNodeSet-seed#[1DC5C60AE231867]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1854804 INFO (qtp104353061-17164) [n:127.0.0.1:47708_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=nodeset_collection&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=http://127.0.0.1:52792/solr,http://127.0.0.1:35324/solr&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1854810 INFO (OverseerThreadFactory-3223-thread-1-processing-n:127.0.0.1:56783_solr) [n:127.0.0.1:56783_solr ] o.a.s.c.CreateCollectionCmd Create collection nodeset_collection
[junit4] 2> 1854812 WARN (OverseerThreadFactory-3223-thread-1-processing-n:127.0.0.1:56783_solr) [n:127.0.0.1:56783_solr ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (nodeset_collection) without cores.
[junit4] 2> 1854995 INFO (qtp104353061-17164) [n:127.0.0.1:47708_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1854995 INFO (qtp104353061-17164) [n:127.0.0.1:47708_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=nodeset_collection&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=http://127.0.0.1:52792/solr,http://127.0.0.1:35324/solr&wt=javabin&version=2} status=0 QTime=191
[junit4] 2> 1854996 INFO (TEST-HdfsCollectionsAPIDistributedZkTest.testCreateNodeSet-seed#[1DC5C60AE231867]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testCreateNodeSet
[junit4] 2> 1855105 INFO (TEST-HdfsCollectionsAPIDistributedZkTest.testMissingRequiredParameters-seed#[1DC5C60AE231867]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testMissingRequiredParameters
[junit4] 2> 1855105 INFO (TEST-HdfsCollectionsAPIDistributedZkTest.testMissingRequiredParameters-seed#[1DC5C60AE231867]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1855106 INFO (qtp104353061-17165) [n:127.0.0.1:47708_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=nodeset_collection&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1855109 INFO (OverseerCollectionConfigSetProcessor-98622705007067144-127.0.0.1:56783_solr-n_0000000000) [n:127.0.0.1:56783_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1855110 INFO (OverseerThreadFactory-3223-thread-2-processing-n:127.0.0.1:56783_solr) [n:127.0.0.1:56783_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 1855886 INFO (qtp104353061-17165) [n:127.0.0.1:47708_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=nodeset_collection&action=DELETE&wt=javabin&version=2} status=0 QTime=779
[junit4] 2> 1855887 INFO (qtp104353061-17166) [n:127.0.0.1:47708_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1855887 ERROR (qtp104353061-17166) [n:127.0.0.1:47708_solr ] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Missing required parameter: name
[junit4] 2> at org.apache.solr.common.params.RequiredSolrParams.getParams(RequiredSolrParams.java:90)
[junit4] 2> at org.apache.solr.common.params.SolrParams.getAll(SolrParams.java:462)
[junit4] 2> at org.apache.solr.common.params.SolrParams.getAll(SolrParams.java:479)
[junit4] 2> at org.apache.
[...truncated too long message...]
le=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=pl-PL -Dtests.timezone=PRC -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] FAILURE 85.6s J0 | SharedFSAutoReplicaFailoverTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: expected:<2> but was:<3>
[junit4] > at __randomizedtesting.SeedInfo.seed([1DC5C60AE231867:898863BA00DF759F]:0)
[junit4] > at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.assertSliceAndReplicaCount(SharedFSAutoReplicaFailoverTest.java:403)
[junit4] > at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:248)
[junit4] > at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3407100 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[1DC5C60AE231867]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 3407114 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[1DC5C60AE231867]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 3407114 WARN (578533353@qtp-882789549-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:48254) [ ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
[junit4] 2> 3407114 WARN (578533353@qtp-882789549-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:48254) [ ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
[junit4] 2> 3407115 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1DC5C60AE231867-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1DC5C60AE231867-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:60517) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
[junit4] 2> 3407115 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1DC5C60AE231867-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1DC5C60AE231867-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:60517) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-1890930609-10.41.0.5-1504864487219 (Datanode Uuid ff1bcd98-7f3e-4e84-a7f1-6be2f8436a44) service to localhost/127.0.0.1:60517
[junit4] 2> 3407117 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[1DC5C60AE231867]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 3407140 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[1DC5C60AE231867]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 3407140 WARN (1010716742@qtp-1449666160-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41994) [ ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
[junit4] 2> 3407143 WARN (1010716742@qtp-1449666160-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41994) [ ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
[junit4] 2> 3407144 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1DC5C60AE231867-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1DC5C60AE231867-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:60517) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
[junit4] 2> 3407144 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1DC5C60AE231867-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1DC5C60AE231867-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:60517) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-1890930609-10.41.0.5-1504864487219 (Datanode Uuid a7808969-a10f-4282-ba7e-3c632cf5ee75) service to localhost/127.0.0.1:60517
[junit4] 2> 3407174 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[1DC5C60AE231867]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 3407174 WARN (1827848810@qtp-1005081840-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34748) [ ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
[junit4] 2> 3407174 WARN (1827848810@qtp-1005081840-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34748) [ ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_1DC5C60AE231867-001
[junit4] 2> Sep 08, 2017 9:56:17 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 35 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), text=PostingsFormat(name=Memory)}, docValues:{_version_=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1329, maxMBSortInHeap=7.489564561220329, sim=RandomSimilarity(queryNorm=false): {}, locale=pl-PL, timezone=PRC
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=115964968,total=522190848
[junit4] 2> NOTE: All tests run in this JVM: [NodeMutatorTest, DistributedFacetPivotLongTailTest, ConnectionReuseTest, ReplicationFactorTest, TestConfigSetsAPI, TestLegacyFieldCache, SpellingQueryConverterTest, TestSolrQueryParser, TestConfig, TestCSVLoader, TestIndexingPerformance, TestDynamicFieldCollectionResource, MoveReplicaHDFSUlogDirTest, HdfsSyncSliceTest, OverseerCollectionConfigSetProcessorTest, TestCloudManagedSchema, LeaderFailureAfterFreshStartTest, TestUnifiedSolrHighlighter, TestPayloadCheckQParserPlugin, BigEndianAscendingWordSerializerTest, FieldAnalysisRequestHandlerTest, TestStressUserVersions, TestRecoveryHdfs, TestDownShardTolerantSearch, TestLeaderElectionWithEmptyReplica, HdfsDirectoryTest, TestSolr4Spatial, IndexBasedSpellCheckerTest, CursorMarkTest, EnumFieldTest, RecoveryAfterSoftCommitTest, SSLMigrationTest, HttpPartitionTest, CursorPagingTest, TestPerFieldSimilarityWithDefaultOverride, TestOmitPositions, HdfsUnloadDistributedZkTest, TestSchemaVersionResource, HdfsChaosMonkeySafeLeaderTest, TestCloudJSONFacetJoinDomain, TestSQLHandler, TestStressLucene, CurrencyFieldTypeTest, OutputWriterTest, TestBlobHandler, TestManagedStopFilterFactory, UpdateLogTest, XmlUpdateRequestHandlerTest, TestLegacyFieldReuse, TestDistributedSearch, TestFastOutputStream, TestReplicationHandler, DistributedQueryElevationComponentTest, TestSizeLimitedDistributedMap, TestFileDictionaryLookup, SynonymTokenizerTest, TestJoin, ChaosMonkeyNothingIsSafeTest, UpdateParamsTest, TestNestedDocsSort, SolrIndexSplitterTest, TestExtendedDismaxParser, TestMultiValuedNumericRangeQuery, DefaultValueUpdateProcessorTest, DistanceUnitsTest, PropertiesRequestHandlerTest, MBeansHandlerTest, DistributedSuggestComponentTest, DebugComponentTest, ConcurrentDeleteAndCreateCollectionTest, TestManagedSchemaAPI, TestRTimerTree, DistributedFacetPivotWhiteBoxTest, TestJavabinTupleStreamParser, TestXmlQParserPlugin, BadCopyFieldTest, BlockCacheTest, TestIBSimilarityFactory, ReplicaListTransformerTest, SolrSlf4jReporterTest, TestEmbeddedSolrServerConstructors, TestTrackingShardHandlerFactory, TestLegacyNumericRangeQueryBuilder, BlockJoinFacetSimpleTest, SolrPluginUtilsTest, TestRealTimeGet, HdfsBasicDistributedZk2Test, SolrIndexConfigTest, TestNumericTokenStream, TestMinMaxOnMultiValuedField, SuggestComponentContextFilterQueryTest, TestNamedUpdateProcessors, TestUninvertingReader, BasicZkTest, TestSSLRandomization, UUIDFieldTest, LeaderElectionIntegrationTest, TestReplicaProperties, SolrMetricReporterTest, TestAuthenticationFramework, TestSolrCoreParser, TestCSVResponseWriter, RequestHandlersTest, TestCollationField, TestCustomSort, SpellCheckCollatorWithCollapseTest, SubstringBytesRefFilterTest, DistributedVersionInfoTest, TestFieldResource, TestConfigsApi, MigrateRouteKeyTest, TestRandomFlRTGCloud, SharedFSAutoReplicaFailoverTest]
[junit4] Completed [721/729 (5!)] on J0 in 100.34s, 1 test, 1 failure <<< FAILURES!
[...truncated 43345 lines...]
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 39 - Still
unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/39/
3 tests failed.
FAILED: org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test.test
Error Message:
.response.numFound:0!=1
Stack Trace:
junit.framework.AssertionFailedError: .response.numFound:0!=1
at __randomizedtesting.SeedInfo.seed([69D7EF372BB13B66:E183D0ED854D569E]:0)
at junit.framework.Assert.fail(Assert.java:50)
at org.apache.solr.BaseDistributedSearchTestCase.compareSolrResponses(BaseDistributedSearchTestCase.java:913)
at org.apache.solr.BaseDistributedSearchTestCase.compareResponses(BaseDistributedSearchTestCase.java:940)
at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:607)
at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:587)
at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:566)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkQueries(AbstractFullDistribZkTestBase.java:1050)
at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:102)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.HttpPartitionTest.test
Error Message:
Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: { "collMinRf_1x3":{ "pullReplicas":"0", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{ "core_node1":{ "core":"collMinRf_1x3_shard1_replica_t3", "base_url":"http://127.0.0.1:56032/m/o", "node_name":"127.0.0.1:56032_m%2Fo", "state":"active", "type":"TLOG"}, "core_node2":{ "core":"collMinRf_1x3_shard1_replica_t1", "base_url":"http://127.0.0.1:53584/m/o", "node_name":"127.0.0.1:53584_m%2Fo", "state":"active", "type":"TLOG", "leader":"true"}, "core_node3":{ "core":"collMinRf_1x3_shard1_replica_t2", "base_url":"http://127.0.0.1:38128/m/o", "node_name":"127.0.0.1:38128_m%2Fo", "state":"active", "type":"TLOG"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"0", "tlogReplicas":"3"}, "collection1":{ "pullReplicas":"0", "replicationFactor":"1", "shards":{ "shard1":{ "range":"80000000-ffffffff", "state":"active", "replicas":{"core_node2":{ "core":"collection1_shard1_replica_t0", "base_url":"http://127.0.0.1:53584/m/o", "node_name":"127.0.0.1:53584_m%2Fo", "state":"active", "type":"TLOG", "leader":"true"}}}, "shard2":{ "range":"0-7fffffff", "state":"active", "replicas":{ "core_node1":{ "core":"collection1_shard2_replica_t0", "base_url":"http://127.0.0.1:56032/m/o", "node_name":"127.0.0.1:56032_m%2Fo", "state":"active", "type":"TLOG", "leader":"true"}, "core_node3":{ "core":"collection1_shard2_replica_t1", "base_url":"http://127.0.0.1:41484/m/o", "node_name":"127.0.0.1:41484_m%2Fo", "state":"active", "type":"TLOG"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"1", "tlogReplicas":"0"}, "control_collection":{ "pullReplicas":"0", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{"core_node1":{ "core":"control_collection_shard1_replica_n1", "base_url":"http://127.0.0.1:38128/m/o", "node_name":"127.0.0.1:38128_m%2Fo", "state":"active", "type":"NRT", "leader":"true"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"1", "tlogReplicas":"0"}, "c8n_1x2":{ "pullReplicas":"0", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{ "core_node1":{ "core":"c8n_1x2_shard1_replica_t1", "base_url":"http://127.0.0.1:56032/m/o", "node_name":"127.0.0.1:56032_m%2Fo", "state":"active", "type":"TLOG", "leader":"true"}, "core_node2":{ "state":"down", "base_url":"http://127.0.0.1:38128/m/o", "core":"c8n_1x2_shard1_replica_t2", "node_name":"127.0.0.1:38128_m%2Fo", "type":"TLOG"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"0", "tlogReplicas":"2"}}
Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: {
"collMinRf_1x3":{
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{
"core_node1":{
"core":"collMinRf_1x3_shard1_replica_t3",
"base_url":"http://127.0.0.1:56032/m/o",
"node_name":"127.0.0.1:56032_m%2Fo",
"state":"active",
"type":"TLOG"},
"core_node2":{
"core":"collMinRf_1x3_shard1_replica_t1",
"base_url":"http://127.0.0.1:53584/m/o",
"node_name":"127.0.0.1:53584_m%2Fo",
"state":"active",
"type":"TLOG",
"leader":"true"},
"core_node3":{
"core":"collMinRf_1x3_shard1_replica_t2",
"base_url":"http://127.0.0.1:38128/m/o",
"node_name":"127.0.0.1:38128_m%2Fo",
"state":"active",
"type":"TLOG"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"0",
"tlogReplicas":"3"},
"collection1":{
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{
"shard1":{
"range":"80000000-ffffffff",
"state":"active",
"replicas":{"core_node2":{
"core":"collection1_shard1_replica_t0",
"base_url":"http://127.0.0.1:53584/m/o",
"node_name":"127.0.0.1:53584_m%2Fo",
"state":"active",
"type":"TLOG",
"leader":"true"}}},
"shard2":{
"range":"0-7fffffff",
"state":"active",
"replicas":{
"core_node1":{
"core":"collection1_shard2_replica_t0",
"base_url":"http://127.0.0.1:56032/m/o",
"node_name":"127.0.0.1:56032_m%2Fo",
"state":"active",
"type":"TLOG",
"leader":"true"},
"core_node3":{
"core":"collection1_shard2_replica_t1",
"base_url":"http://127.0.0.1:41484/m/o",
"node_name":"127.0.0.1:41484_m%2Fo",
"state":"active",
"type":"TLOG"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"1",
"tlogReplicas":"0"},
"control_collection":{
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{"core_node1":{
"core":"control_collection_shard1_replica_n1",
"base_url":"http://127.0.0.1:38128/m/o",
"node_name":"127.0.0.1:38128_m%2Fo",
"state":"active",
"type":"NRT",
"leader":"true"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"1",
"tlogReplicas":"0"},
"c8n_1x2":{
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{
"core_node1":{
"core":"c8n_1x2_shard1_replica_t1",
"base_url":"http://127.0.0.1:56032/m/o",
"node_name":"127.0.0.1:56032_m%2Fo",
"state":"active",
"type":"TLOG",
"leader":"true"},
"core_node2":{
"state":"down",
"base_url":"http://127.0.0.1:38128/m/o",
"core":"c8n_1x2_shard1_replica_t2",
"node_name":"127.0.0.1:38128_m%2Fo",
"type":"TLOG"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"0",
"tlogReplicas":"2"}}
at __randomizedtesting.SeedInfo.seed([69D7EF372BB13B66:E183D0ED854D569E]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1996)
at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:368)
at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Timeout waiting for all live and active
Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
at __randomizedtesting.SeedInfo.seed([69D7EF372BB13B66:E183D0ED854D569E]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11658 lines...]
[junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/init-core-data-001
[junit4] 2> 658351 WARN (SUITE-HttpPartitionTest-seed#[69D7EF372BB13B66]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=10 numCloses=10
[junit4] 2> 658352 INFO (SUITE-HttpPartitionTest-seed#[69D7EF372BB13B66]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 658363 INFO (SUITE-HttpPartitionTest-seed#[69D7EF372BB13B66]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 658363 INFO (SUITE-HttpPartitionTest-seed#[69D7EF372BB13B66]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /m/o
[junit4] 2> 658371 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 658379 INFO (Thread-628) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 658379 INFO (Thread-628) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 658477 ERROR (Thread-628) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 658747 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.ZkTestServer start zk server on port:52997
[junit4] 2> 658838 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 658839 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 658841 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 658842 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 658843 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 658843 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 658844 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 658845 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 658845 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 658846 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 658847 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 658848 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 659388 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 659389 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5c236413{/m/o,null,AVAILABLE}
[junit4] 2> 659390 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@752de79e{HTTP/1.1,[http/1.1]}{127.0.0.1:49023}
[junit4] 2> 659390 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.Server Started @667954ms
[junit4] 2> 659390 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/tempDir-001/control/data, replicaType=NRT, hostContext=/m/o, hostPort=38128, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/control-001/cores}
[junit4] 2> 659390 ERROR (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 659390 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 659390 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 659390 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 659390 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-07T07:05:47.555Z
[junit4] 2> 659396 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 659396 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/control-001/solr.xml
[junit4] 2> 659414 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 659436 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52997/solr
[junit4] 2> 659756 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 659757 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38128_m%2Fo
[junit4] 2> 659757 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.Overseer Overseer (id=98616472149360644-127.0.0.1:38128_m%2Fo-n_0000000000) starting
[junit4] 2> 659813 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38128_m%2Fo
[junit4] 2> 659829 INFO (OverseerStateUpdate-98616472149360644-127.0.0.1:38128_m%2Fo-n_0000000000) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 659931 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
[junit4] 2> 659959 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
[junit4] 2> 659959 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
[junit4] 2> 659961 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/control-001/cores
[junit4] 2> 660028 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 660028 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52997/solr ready
[junit4] 2> 660029 INFO (SocketProxy-Acceptor-38128) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=57347,localport=38128], receiveBufferSize:531000
[junit4] 2> 660122 INFO (SocketProxy-Acceptor-38128) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=49023,localport=60031], receiveBufferSize=530904
[junit4] 2> 660130 INFO (qtp319343565-3576) [n:127.0.0.1:38128_m%2Fo ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38128_m%252Fo&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 660147 INFO (OverseerThreadFactory-1446-thread-1-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 660272 INFO (SocketProxy-Acceptor-38128) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=57351,localport=38128], receiveBufferSize:531000
[junit4] 2> 660281 INFO (SocketProxy-Acceptor-38128) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=49023,localport=60035], receiveBufferSize=530904
[junit4] 2> 660289 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 660289 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 660393 INFO (zkCallback-713-thread-1-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 661364 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 661465 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 661818 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 661834 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 661834 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
[junit4] 2> 661834 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 661834 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 661848 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.22171267237873948]
[junit4] 2> 661850 WARN (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 661935 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 661935 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 661970 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 661970 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 661972 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=9.2421875, floorSegmentMB=1.416015625, forceMergeDeletesPctAllowed=7.259038767803224, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 661972 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@242e154[control_collection_shard1_replica_n1] main]
[junit4] 2> 661973 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 661973 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 661974 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 661975 INFO (searcherExecutor-1449-thread-1-processing-n:127.0.0.1:38128_m%2Fo x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@242e154[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 661975 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577863558086000640
[junit4] 2> 661988 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 661988 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 661988 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38128/m/o/control_collection_shard1_replica_n1/
[junit4] 2> 661988 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 661988 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:38128/m/o/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 661988 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 661991 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38128/m/o/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 662185 INFO (zkCallback-713-thread-1-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 662233 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 662237 INFO (qtp319343565-3574) [n:127.0.0.1:38128_m%2Fo c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1948
[junit4] 2> 662413 INFO (qtp319343565-3576) [n:127.0.0.1:38128_m%2Fo ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 662513 INFO (zkCallback-713-thread-1-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 663413 INFO (qtp319343565-3576) [n:127.0.0.1:38128_m%2Fo ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:38128_m%252Fo&wt=javabin&version=2} status=0 QTime=3283
[junit4] 2> 663499 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 663500 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52997/solr ready
[junit4] 2> 663500 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 663501 INFO (SocketProxy-Acceptor-38128) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=57361,localport=38128], receiveBufferSize:531000
[junit4] 2> 663512 INFO (SocketProxy-Acceptor-38128) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=49023,localport=60045], receiveBufferSize=530904
[junit4] 2> 663520 INFO (qtp319343565-3577) [n:127.0.0.1:38128_m%2Fo ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 663539 INFO (OverseerCollectionConfigSetProcessor-98616472149360644-127.0.0.1:38128_m%2Fo-n_0000000000) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 663540 INFO (OverseerThreadFactory-1446-thread-2-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 663540 WARN (OverseerThreadFactory-1446-thread-2-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 663927 INFO (qtp319343565-3577) [n:127.0.0.1:38128_m%2Fo ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 663927 INFO (qtp319343565-3577) [n:127.0.0.1:38128_m%2Fo ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=407
[junit4] 2> 664323 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-1-001 of type TLOG
[junit4] 2> 664324 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 664411 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@23303f8c{/m/o,null,AVAILABLE}
[junit4] 2> 664411 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@663eb2d5{HTTP/1.1,[http/1.1]}{127.0.0.1:57009}
[junit4] 2> 664411 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.Server Started @672975ms
[junit4] 2> 664411 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/m/o, hostPort=56032, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-1-001/cores}
[junit4] 2> 664412 ERROR (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 664412 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 664412 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 664412 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 664412 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-07T07:05:52.577Z
[junit4] 2> 664439 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 664439 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-1-001/solr.xml
[junit4] 2> 664444 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 664464 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52997/solr
[junit4] 2> 664983 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:56032_m%2Fo ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 664984 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:56032_m%2Fo ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 664986 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:56032_m%2Fo ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56032_m%2Fo
[junit4] 2> 664987 INFO (zkCallback-720-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 664987 INFO (zkCallback-713-thread-1-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 664992 INFO (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 665233 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:56032_m%2Fo ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
[junit4] 2> 665244 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:56032_m%2Fo ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
[junit4] 2> 665245 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:56032_m%2Fo ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
[junit4] 2> 665246 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:56032_m%2Fo ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-1-001/cores
[junit4] 2> 665312 INFO (SocketProxy-Acceptor-56032) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54042,localport=56032], receiveBufferSize:531000
[junit4] 2> 665332 INFO (SocketProxy-Acceptor-56032) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=57009,localport=42862], receiveBufferSize=530904
[junit4] 2> 665337 INFO (qtp1554326178-3624) [n:127.0.0.1:56032_m%2Fo ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:56032_m%252Fo&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 665352 INFO (OverseerCollectionConfigSetProcessor-98616472149360644-127.0.0.1:38128_m%2Fo-n_0000000000) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 665353 INFO (OverseerThreadFactory-1446-thread-3-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:56032_m%2Fo for creating new replica
[junit4] 2> 665356 INFO (SocketProxy-Acceptor-56032) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54045,localport=56032], receiveBufferSize:531000
[junit4] 2> 665370 INFO (SocketProxy-Acceptor-56032) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=57009,localport=42865], receiveBufferSize=530904
[junit4] 2> 665375 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 665377 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 665482 INFO (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 666411 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 666468 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.s.IndexSchema [collection1_shard2_replica_t0] Schema name=test
[junit4] 2> 666768 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 666781 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t0' using configuration from collection collection1, trusted=true
[junit4] 2> 666787 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t0' (registry 'solr.core.collection1.shard2.replica_t0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
[junit4] 2> 666787 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 666787 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SolrCore [[collection1_shard2_replica_t0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-1-001/cores/collection1_shard2_replica_t0], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-1-001/cores/collection1_shard2_replica_t0/data/]
[junit4] 2> 666793 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.22171267237873948]
[junit4] 2> 666798 WARN (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 666839 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 666840 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 666841 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 666841 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 666843 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=9.2421875, floorSegmentMB=1.416015625, forceMergeDeletesPctAllowed=7.259038767803224, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 666844 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.s.SolrIndexSearcher Opening [Searcher@3804fd18[collection1_shard2_replica_t0] main]
[junit4] 2> 666845 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 666848 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 666849 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 666849 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577863563196760064
[junit4] 2> 666851 INFO (searcherExecutor-1460-thread-1-processing-n:127.0.0.1:56032_m%2Fo x:collection1_shard2_replica_t0 s:shard2 c:collection1) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SolrCore [collection1_shard2_replica_t0] Registered new searcher Searcher@3804fd18[collection1_shard2_replica_t0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 666858 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 666858 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 666858 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56032/m/o/collection1_shard2_replica_t0/
[junit4] 2> 666858 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 666858 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.SyncStrategy http://127.0.0.1:56032/m/o/collection1_shard2_replica_t0/ has no replicas
[junit4] 2> 666858 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 666858 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ZkController collection1_shard2_replica_t0 stopping background replication from leader
[junit4] 2> 666861 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56032/m/o/collection1_shard2_replica_t0/ shard2
[junit4] 2> 666962 INFO (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 667012 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 667014 INFO (qtp1554326178-3622) [n:127.0.0.1:56032_m%2Fo c:collection1 s:shard2 x:collection1_shard2_replica_t0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1638
[junit4] 2> 667018 INFO (qtp1554326178-3624) [n:127.0.0.1:56032_m%2Fo ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:56032_m%252Fo&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=1680
[junit4] 2> 667158 INFO (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 667360 INFO (OverseerCollectionConfigSetProcessor-98616472149360644-127.0.0.1:38128_m%2Fo-n_0000000000) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 667365 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-2-001 of type TLOG
[junit4] 2> 667366 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 667443 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7046306e{/m/o,null,AVAILABLE}
[junit4] 2> 667444 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@18c9c2ea{HTTP/1.1,[http/1.1]}{127.0.0.1:55541}
[junit4] 2> 667444 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.Server Started @676008ms
[junit4] 2> 667444 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/m/o, hostPort=53584, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-2-001/cores}
[junit4] 2> 667444 ERROR (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 667451 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 667451 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 667451 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 667451 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-07T07:05:55.616Z
[junit4] 2> 667510 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 667510 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-2-001/solr.xml
[junit4] 2> 667519 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 667538 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52997/solr
[junit4] 2> 667749 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:53584_m%2Fo ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 667750 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:53584_m%2Fo ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 667752 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:53584_m%2Fo ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53584_m%2Fo
[junit4] 2> 667753 INFO (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 667753 INFO (zkCallback-713-thread-1-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 667753 INFO (zkCallback-720-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 667771 INFO (zkCallback-731-thread-1-processing-n:127.0.0.1:53584_m%2Fo) [n:127.0.0.1:53584_m%2Fo ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 667881 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:53584_m%2Fo ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
[junit4] 2> 667899 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:53584_m%2Fo ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
[junit4] 2> 667899 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:53584_m%2Fo ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
[junit4] 2> 667900 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:53584_m%2Fo ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-2-001/cores
[junit4] 2> 667934 INFO (SocketProxy-Acceptor-53584) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=34819,localport=53584], receiveBufferSize:531000
[junit4] 2> 667938 INFO (SocketProxy-Acceptor-53584) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=55541,localport=37275], receiveBufferSize=530904
[junit4] 2> 667942 INFO (qtp704117609-3658) [n:127.0.0.1:53584_m%2Fo ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:53584_m%252Fo&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 667961 INFO (OverseerThreadFactory-1446-thread-4-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:53584_m%2Fo for creating new replica
[junit4] 2> 667962 INFO (SocketProxy-Acceptor-53584) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=34822,localport=53584], receiveBufferSize:531000
[junit4] 2> 667966 INFO (SocketProxy-Acceptor-53584) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=55541,localport=37278], receiveBufferSize=530904
[junit4] 2> 667966 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 667966 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 668072 INFO (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 668072 INFO (zkCallback-731-thread-1-processing-n:127.0.0.1:53584_m%2Fo) [n:127.0.0.1:53584_m%2Fo ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 668985 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 669009 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.s.IndexSchema [collection1_shard1_replica_t0] Schema name=test
[junit4] 2> 669391 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 669425 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t0' using configuration from collection collection1, trusted=true
[junit4] 2> 669425 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t0' (registry 'solr.core.collection1.shard1.replica_t0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
[junit4] 2> 669425 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 669425 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SolrCore [[collection1_shard1_replica_t0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-2-001/cores/collection1_shard1_replica_t0], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-2-001/cores/collection1_shard1_replica_t0/data/]
[junit4] 2> 669440 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.22171267237873948]
[junit4] 2> 669448 WARN (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 669540 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 669540 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 669541 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 669541 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 669542 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=9.2421875, floorSegmentMB=1.416015625, forceMergeDeletesPctAllowed=7.259038767803224, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 669542 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.s.SolrIndexSearcher Opening [Searcher@6d6f0057[collection1_shard1_replica_t0] main]
[junit4] 2> 669543 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 669543 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 669544 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 669544 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577863566022672384
[junit4] 2> 669550 INFO (searcherExecutor-1471-thread-1-processing-n:127.0.0.1:53584_m%2Fo x:collection1_shard1_replica_t0 s:shard1 c:collection1) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SolrCore [collection1_shard1_replica_t0] Registered new searcher Searcher@6d6f0057[collection1_shard1_replica_t0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 669552 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 669552 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 669552 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53584/m/o/collection1_shard1_replica_t0/
[junit4] 2> 669832 INFO (zkCallback-731-thread-1-processing-n:127.0.0.1:53584_m%2Fo) [n:127.0.0.1:53584_m%2Fo ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 670062 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 669832 INFO (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 670063 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.SyncStrategy http://127.0.0.1:53584/m/o/collection1_shard1_replica_t0/ has no replicas
[junit4] 2> 670063 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 670063 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.ZkController collection1_shard1_replica_t0 stopping background replication from leader
[junit4] 2> 670339 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53584/m/o/collection1_shard1_replica_t0/ shard1
[junit4] 2> 670441 INFO (zkCallback-731-thread-1-processing-n:127.0.0.1:53584_m%2Fo) [n:127.0.0.1:53584_m%2Fo ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 670441 INFO (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 670490 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 670492 INFO (qtp704117609-3656) [n:127.0.0.1:53584_m%2Fo c:collection1 s:shard1 x:collection1_shard1_replica_t0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2526
[junit4] 2> 670494 INFO (qtp704117609-3658) [n:127.0.0.1:53584_m%2Fo ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:53584_m%252Fo&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=2551
[junit4] 2> 670711 INFO (zkCallback-731-thread-1-processing-n:127.0.0.1:53584_m%2Fo) [n:127.0.0.1:53584_m%2Fo ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 670711 INFO (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 670934 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-3-001 of type TLOG
[junit4] 2> 670935 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 670941 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@642ac4f4{/m/o,null,AVAILABLE}
[junit4] 2> 670941 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@556207f0{HTTP/1.1,[http/1.1]}{127.0.0.1:39720}
[junit4] 2> 670941 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.e.j.s.Server Started @679505ms
[junit4] 2> 670941 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/tempDir-001/jetty3, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/m/o, hostPort=41484, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-3-001/cores}
[junit4] 2> 670942 ERROR (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 670943 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 670943 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 670943 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 670943 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-07T07:05:59.108Z
[junit4] 2> 670976 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 670976 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-3-001/solr.xml
[junit4] 2> 671022 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 671027 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52997/solr
[junit4] 2> 671316 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:41484_m%2Fo ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 671317 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:41484_m%2Fo ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 671319 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:41484_m%2Fo ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41484_m%2Fo
[junit4] 2> 671330 INFO (zkCallback-737-thread-1-processing-n:127.0.0.1:41484_m%2Fo) [n:127.0.0.1:41484_m%2Fo ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 671347 INFO (zkCallback-720-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 671347 INFO (zkCallback-725-thread-1-processing-n:127.0.0.1:56032_m%2Fo) [n:127.0.0.1:56032_m%2Fo ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 671348 INFO (zkCallback-731-thread-1-processing-n:127.0.0.1:53584_m%2Fo) [n:127.0.0.1:53584_m%2Fo ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 671348 INFO (zkCallback-713-thread-1-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 671442 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:41484_m%2Fo ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
[junit4] 2> 671459 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:41484_m%2Fo ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
[junit4] 2> 671459 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:41484_m%2Fo ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a68c280
[junit4] 2> 671464 INFO (TEST-HttpPartitionTest.test-seed#[69D7EF372BB13B66]) [n:127.0.0.1:41484_m%2Fo ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_69D7EF372BB13B66-001/shard-3-001/cores
[junit4] 2> 671554 INFO (qtp704117609-3659) [n:127.0.0.1:53584_m%2Fo ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41484_m%252Fo&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 671572 INFO (OverseerCollectionConfigSetProcessor-98616472149360644-127.0.0.1:38128_m%2Fo-n_0000000000) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 671727 INFO (OverseerThreadFactory-1446-thread-5-processing-n:127.0.0.1:38128_m%2Fo) [n:127.0.0.1:38128_m%2Fo ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:41484_m%2Fo for creating new replica
[junit4] 2> 671728 INFO (SocketProxy-Acceptor-41484) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=33479,localport=41484], receiveBufferSize:531000
[junit4] 2> 671751 INFO (SocketProxy-Acceptor-41484) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=39720,localport=37358], receiveBufferSize=530904
[junit4] 2> 671758 INFO (qtp706179858-3692) [n:127.0.0.1:41484_m%2Fo ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t1&action=CREATE&col
[...truncated too long message...]
er IncrementalBlockReportManager interrupted
[junit4] 2> 5910593 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_69D7EF372BB13B66-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_69D7EF372BB13B66-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:59049) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-761140916-10.41.0.5-1504773134652 (Datanode Uuid 44d47a55-e14e-402f-b52c-e6c5b61dfb45) service to localhost/127.0.0.1:59049
[junit4] 2> 5910602 WARN (SUITE-HdfsBasicDistributedZk2Test-seed#[69D7EF372BB13B66]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 5910635 INFO (SUITE-HdfsBasicDistributedZk2Test-seed#[69D7EF372BB13B66]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 5910635 WARN (1440079205@qtp-938187338-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:54992) [ ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
[junit4] 2> 5910636 WARN (1440079205@qtp-938187338-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:54992) [ ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
[junit4] 2> 5910740 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_69D7EF372BB13B66-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_69D7EF372BB13B66-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:59049) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
[junit4] 2> 5910740 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_69D7EF372BB13B66-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_69D7EF372BB13B66-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:59049) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-761140916-10.41.0.5-1504773134652 (Datanode Uuid 096f87eb-da2f-4576-b538-7114bfdff4f9) service to localhost/127.0.0.1:59049
[junit4] 2> 5910785 INFO (SUITE-HdfsBasicDistributedZk2Test-seed#[69D7EF372BB13B66]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 5910785 WARN (68133024@qtp-1610702844-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34307) [ ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
[junit4] 2> 5910785 WARN (68133024@qtp-1610702844-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34307) [ ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_69D7EF372BB13B66-001
[junit4] 2> Sep 07, 2017 8:33:15 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {rnd_b=BlockTreeOrds(blocksize=128), SubjectTerms_mfacet=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), oddField_s=PostingsFormat(name=LuceneVarGapFixedInterval), multiDefault=PostingsFormat(name=Memory), a_t=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), foo_b=PostingsFormat(name=LuceneVarGapFixedInterval), id=BlockTreeOrds(blocksize=128), text=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{other_tl1=DocValuesFormat(name=Memory), range_facet_l_dv=DocValuesFormat(name=Asserting), n_l1=DocValuesFormat(name=Memory), intDefault=DocValuesFormat(name=Memory), n_dt1=DocValuesFormat(name=Lucene70), a_i1=DocValuesFormat(name=Memory), n_td1=DocValuesFormat(name=Lucene70), n_d1=DocValuesFormat(name=Memory), range_facet_l=DocValuesFormat(name=Memory), n_f1=DocValuesFormat(name=Lucene70), n_ti1=DocValuesFormat(name=Asserting), n_tl1=DocValuesFormat(name=Lucene70), _version_=DocValuesFormat(name=Memory), n_tf1=DocValuesFormat(name=Memory), n_tdt1=DocValuesFormat(name=Memory), id_i1=DocValuesFormat(name=Lucene70), foo_d=DocValuesFormat(name=Asserting), range_facet_i_dv=DocValuesFormat(name=Memory), intDvoDefault=DocValuesFormat(name=Direct), foo_f=DocValuesFormat(name=Direct), timestamp=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1413, maxMBSortInHeap=5.826986203817525, sim=RandomSimilarity(queryNorm=true): {}, locale=ga-IE, timezone=Asia/Dili
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=152052264,total=504365056
[junit4] 2> NOTE: All tests run in this JVM: [DistributedFacetExistsSmallTest, TestSolrCLIRunExample, DistributedIntervalFacetingTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, CdcrVersionReplicationTest, OpenCloseCoreStressTest, DistribJoinFromCollectionTest, CheckHdfsIndexTest, SampleTest, SolrCoreMetricManagerTest, TlogReplayBufferedWhileIndexingTest, TestSolrQueryResponse, TestPostingsSolrHighlighter, AddBlockUpdateTest, TestUpdate, TestCoreContainer, StatelessScriptUpdateProcessorFactoryTest, TestDynamicFieldResource, TestDFISimilarityFactory, SystemInfoHandlerTest, DistributedMLTComponentTest, ZkStateReaderTest, TestMaxScoreQueryParser, SaslZkACLProviderTest, TestInitParams, DocValuesNotIndexedTest, AtomicUpdatesTest, BlockJoinFacetDistribTest, TestStressVersions, TestSubQueryTransformerCrossCore, TestSerializedLuceneMatchVersion, MissingSegmentRecoveryTest, MoreLikeThisHandlerTest, CustomCollectionTest, TestJsonFacetRefinement, TestScoreJoinQPNoScore, TestPullReplica, LukeRequestHandlerTest, TestStressRecovery, SpellCheckComponentTest, TestHashQParserPlugin, TestRandomRequestDistribution, TestManagedSynonymFilterFactory, NoCacheHeaderTest, RemoteQueryErrorTest, FacetPivotSmallTest, SpatialRPTFieldTypeTest, DistributedQueryComponentCustomSortTest, SolrCloudExampleTest, ClassificationUpdateProcessorTest, TestLRUCache, TestLeaderElectionZkExpiry, TestFastLRUCache, HighlighterTest, HdfsCollectionsAPIDistributedZkTest, MetricUtilsTest, TestOrdValues, CloudMLTQParserTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestRTGBase, TestSha256AuthenticationProvider, TestSolrCloudWithHadoopAuthPlugin, TestSimpleTrackingShardHandler, SchemaVersionSpecificBehaviorTest, DataDrivenBlockJoinTest, SortSpecParsingTest, ConvertedLegacyTest, TestLockTree, LoggingHandlerTest, TestPerFieldSimilarity, TestNumericRangeQuery32, TestFieldSortValues, FileUtilsTest, BasicFunctionalityTest, TestExclusionRuleCollectionAccess, ShufflingReplicaListTransformerTest, FileBasedSpellCheckerTest, TestMaxTokenLenTokenizer, DirectSolrConnectionTest, AlternateDirectoryTest, SolrJmxReporterCloudTest, TestXIncludeConfig, RuleEngineTest, TestReloadDeadlock, ManagedSchemaRoundRobinCloudTest, SolrCloudReportersTest, TestManagedResourceStorage, CollectionsAPIDistributedZkTest, TestSolrCloudWithDelegationTokens, TestValueSourceCache, JavabinLoaderTest, TestSearchPerf, CollectionsAPIAsyncDistributedZkTest, TestSmileRequest, TestCloudPivotFacet, TestClassNameShortening, HdfsBasicDistributedZkTest, TestStressCloudBlindAtomicUpdates, AsyncCallRequestStatusResponseTest, SliceStateTest, HdfsWriteToMultipleCollectionsTest, DistributedFacetPivotSmallAdvancedTest, ActionThrottleTest, AnalyticsMergeStrategyTest, TestSQLHandlerNonCloud, CdcrReplicationDistributedZkTest, TestClusterProperties, CSVRequestHandlerTest, IndexSchemaRuntimeFieldTest, TestAuthorizationFramework, TestCharFilters, AutoScalingHandlerTest, HLLSerializationTest, TestConfigOverlay, TestLegacyNumericUtils, BooleanFieldTest, TestFieldTypeCollectionResource, TestQueryTypes, ScriptEngineTest, TestHalfAndHalfDocValues, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, LeaderElectionIntegrationTest, ShardRoutingTest, RecoveryZkTest, ClusterStateUpdateTest, TestRandomFaceting, LeaderElectionTest, ZkSolrClientTest, ZkCLITest, TestDistributedSearch, ChaosMonkeySafeLeaderWithPullReplicasTest, CreateCollectionCleanupTest, DeleteLastCustomShardedReplicaTest, DeleteReplicaTest, DistributedQueueTest, DistributedVersionInfoTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, MoveReplicaHDFSTest, MoveReplicaTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerModifyCollectionTest, OverseerRolesTest, OverseerTaskQueueTest, ReplicationFactorTest, RestartWhileUpdatingTest, SharedFSAutoReplicaFailoverUtilsTest, TestCloudDeleteByQuery, TestCloudInspectUtil, TestCloudJSONFacetJoinDomain, TestConfigSetsAPI, TestConfigSetsAPIExclusivity, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestLeaderElectionWithEmptyReplica, TestLeaderInitiatedRecoveryThread, TestMiniSolrCloudClusterSSL, TestRandomFlRTGCloud, TestRequestForwarding, TestSegmentSorting, TestStressInPlaceUpdates, TestTlogReplica, TestTolerantUpdateProcessorRandomCloud, HdfsBasicDistributedZk2Test]
[junit4] Completed [560/729 (3!)] on J1 in 69.81s, 1 test, 1 failure <<< FAILURES!
[...truncated 43843 lines...]
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 38 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/38/
6 tests failed.
FAILED: org.apache.lucene.search.TestInetAddressRangeQueries.testRandomBig
Error Message:
Java heap space
Stack Trace:
java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([A03631D71003B06D:27614C58815ACCED]:0)
at org.apache.lucene.util.bkd.HeapPointWriter.writePackedValue(HeapPointWriter.java:107)
at org.apache.lucene.util.bkd.HeapPointWriter.append(HeapPointWriter.java:128)
at org.apache.lucene.util.bkd.PointReader.split(PointReader.java:68)
at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1791)
at org.apache.lucene.util.bkd.BKDWriter.finish(BKDWriter.java:1008)
at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:141)
at org.apache.lucene.codecs.asserting.AssertingPointsFormat$AssertingPointsWriter.writeField(AssertingPointsFormat.java:137)
at org.apache.lucene.index.PointValuesWriter.flush(PointValuesWriter.java:183)
at org.apache.lucene.index.DefaultIndexingChain.writePoints(DefaultIndexingChain.java:203)
at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:138)
at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:451)
at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:542)
at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:399)
at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:502)
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1717)
at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1462)
at org.apache.lucene.search.BaseRangeFieldQueryTestCase.verify(BaseRangeFieldQueryTestCase.java:190)
at org.apache.lucene.search.BaseRangeFieldQueryTestCase.doTestRandom(BaseRangeFieldQueryTestCase.java:160)
at org.apache.lucene.search.BaseRangeFieldQueryTestCase.testRandomBig(BaseRangeFieldQueryTestCase.java:75)
at org.apache.lucene.search.TestInetAddressRangeQueries.testRandomBig(TestInetAddressRangeQueries.java:81)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
FAILED: org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([16701B8549ADDA26]:0)
FAILED: junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([16701B8549ADDA26]:0)
FAILED: org.apache.solr.cloud.HttpPartitionTest.test
Error Message:
Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: { "collMinRf_1x3":{ "pullReplicas":"0", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{ "core_node1":{ "core":"collMinRf_1x3_shard1_replica_n2", "base_url":"http://127.0.0.1:39733/ggth/x", "node_name":"127.0.0.1:39733_ggth%2Fx", "state":"active", "type":"NRT"}, "core_node2":{ "core":"collMinRf_1x3_shard1_replica_n1", "base_url":"http://127.0.0.1:36295/ggth/x", "node_name":"127.0.0.1:36295_ggth%2Fx", "state":"active", "type":"NRT"}, "core_node3":{ "core":"collMinRf_1x3_shard1_replica_n3", "base_url":"http://127.0.0.1:46319/ggth/x", "node_name":"127.0.0.1:46319_ggth%2Fx", "state":"active", "type":"NRT", "leader":"true"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"3", "tlogReplicas":"0"}, "collection1":{ "pullReplicas":"0", "replicationFactor":"1", "shards":{ "shard1":{ "range":"80000000-ffffffff", "state":"active", "replicas":{"core_node2":{ "core":"collection1_shard1_replica_n0", "base_url":"http://127.0.0.1:39733/ggth/x", "node_name":"127.0.0.1:39733_ggth%2Fx", "state":"active", "type":"NRT", "leader":"true"}}}, "shard2":{ "range":"0-7fffffff", "state":"active", "replicas":{ "core_node1":{ "core":"collection1_shard2_replica_n0", "base_url":"http://127.0.0.1:39622/ggth/x", "node_name":"127.0.0.1:39622_ggth%2Fx", "state":"active", "type":"NRT", "leader":"true"}, "core_node3":{ "core":"collection1_shard2_replica_n1", "base_url":"http://127.0.0.1:46319/ggth/x", "node_name":"127.0.0.1:46319_ggth%2Fx", "state":"active", "type":"NRT"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"1", "tlogReplicas":"0"}, "control_collection":{ "pullReplicas":"0", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{"core_node1":{ "core":"control_collection_shard1_replica_n1", "base_url":"http://127.0.0.1:36295/ggth/x", "node_name":"127.0.0.1:36295_ggth%2Fx", "state":"active", "type":"NRT", "leader":"true"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"1", "tlogReplicas":"0"}, "c8n_1x2":{ "pullReplicas":"0", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{ "core_node1":{ "core":"c8n_1x2_shard1_replica_n2", "base_url":"http://127.0.0.1:39733/ggth/x", "node_name":"127.0.0.1:39733_ggth%2Fx", "state":"active", "type":"NRT", "leader":"true"}, "core_node2":{ "state":"down", "base_url":"http://127.0.0.1:36295/ggth/x", "core":"c8n_1x2_shard1_replica_n1", "node_name":"127.0.0.1:36295_ggth%2Fx", "type":"NRT"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"2", "tlogReplicas":"0"}}
Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x2 come up within 90000 ms! ClusterState: {
"collMinRf_1x3":{
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{
"core_node1":{
"core":"collMinRf_1x3_shard1_replica_n2",
"base_url":"http://127.0.0.1:39733/ggth/x",
"node_name":"127.0.0.1:39733_ggth%2Fx",
"state":"active",
"type":"NRT"},
"core_node2":{
"core":"collMinRf_1x3_shard1_replica_n1",
"base_url":"http://127.0.0.1:36295/ggth/x",
"node_name":"127.0.0.1:36295_ggth%2Fx",
"state":"active",
"type":"NRT"},
"core_node3":{
"core":"collMinRf_1x3_shard1_replica_n3",
"base_url":"http://127.0.0.1:46319/ggth/x",
"node_name":"127.0.0.1:46319_ggth%2Fx",
"state":"active",
"type":"NRT",
"leader":"true"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"3",
"tlogReplicas":"0"},
"collection1":{
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{
"shard1":{
"range":"80000000-ffffffff",
"state":"active",
"replicas":{"core_node2":{
"core":"collection1_shard1_replica_n0",
"base_url":"http://127.0.0.1:39733/ggth/x",
"node_name":"127.0.0.1:39733_ggth%2Fx",
"state":"active",
"type":"NRT",
"leader":"true"}}},
"shard2":{
"range":"0-7fffffff",
"state":"active",
"replicas":{
"core_node1":{
"core":"collection1_shard2_replica_n0",
"base_url":"http://127.0.0.1:39622/ggth/x",
"node_name":"127.0.0.1:39622_ggth%2Fx",
"state":"active",
"type":"NRT",
"leader":"true"},
"core_node3":{
"core":"collection1_shard2_replica_n1",
"base_url":"http://127.0.0.1:46319/ggth/x",
"node_name":"127.0.0.1:46319_ggth%2Fx",
"state":"active",
"type":"NRT"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"1",
"tlogReplicas":"0"},
"control_collection":{
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{"core_node1":{
"core":"control_collection_shard1_replica_n1",
"base_url":"http://127.0.0.1:36295/ggth/x",
"node_name":"127.0.0.1:36295_ggth%2Fx",
"state":"active",
"type":"NRT",
"leader":"true"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"1",
"tlogReplicas":"0"},
"c8n_1x2":{
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{
"core_node1":{
"core":"c8n_1x2_shard1_replica_n2",
"base_url":"http://127.0.0.1:39733/ggth/x",
"node_name":"127.0.0.1:39733_ggth%2Fx",
"state":"active",
"type":"NRT",
"leader":"true"},
"core_node2":{
"state":"down",
"base_url":"http://127.0.0.1:36295/ggth/x",
"core":"c8n_1x2_shard1_replica_n1",
"node_name":"127.0.0.1:36295_ggth%2Fx",
"type":"NRT"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"2",
"tlogReplicas":"0"}}
at __randomizedtesting.SeedInfo.seed([C56A96109C8138C1:4D3EA9CA327D5539]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1996)
at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:368)
at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv
Error Message:
java.lang.RuntimeException: Error from server at http://127.0.0.1:45746/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:34849/solr/test_col_shard5_replica_n1: Server Error request: http://127.0.0.1:34849/solr/test_col_shard5_replica_n1/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A45746%2Fsolr%2Ftest_col_shard2_replica_n2%2F&wt=javabin&version=2 Remote error message: Failed synchronous update on shard StdNode: http://127.0.0.1:36830/solr/test_col_shard5_replica_n2/ update: org.apache.solr.client.solrj.request.UpdateRequest@6dc5b03f
Stack Trace:
java.util.concurrent.ExecutionException: java.lang.RuntimeException: Error from server at http://127.0.0.1:45746/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:34849/solr/test_col_shard5_replica_n1: Server Error
request: http://127.0.0.1:34849/solr/test_col_shard5_replica_n1/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A45746%2Fsolr%2Ftest_col_shard2_replica_n2%2F&wt=javabin&version=2
Remote error message: Failed synchronous update on shard StdNode: http://127.0.0.1:36830/solr/test_col_shard5_replica_n2/ update: org.apache.solr.client.solrj.request.UpdateRequest@6dc5b03f
at __randomizedtesting.SeedInfo.seed([C56A96109C8138C1:F37EF45616DC02D0]:0)
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.checkField(TestStressCloudBlindAtomicUpdates.java:283)
at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates.test_dv(TestStressCloudBlindAtomicUpdates.java:195)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Error from server at http://127.0.0.1:45746/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:34849/solr/test_col_shard5_replica_n1: Server Error
request: http://127.0.0.1:34849/solr/test_col_shard5_replica_n1/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A45746%2Fsolr%2Ftest_col_shard2_replica_n2%2F&wt=javabin&version=2
Remote error message: Failed synchronous update on shard StdNode: http://127.0.0.1:36830/solr/test_col_shard5_replica_n2/ update: org.apache.solr.client.solrj.request.UpdateRequest@6dc5b03f
at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates$Worker.run(TestStressCloudBlindAtomicUpdates.java:411)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:45746/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:34849/solr/test_col_shard5_replica_n1: Server Error
request: http://127.0.0.1:34849/solr/test_col_shard5_replica_n1/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A45746%2Fsolr%2Ftest_col_shard2_replica_n2%2F&wt=javabin&version=2
Remote error message: Failed synchronous update on shard StdNode: http://127.0.0.1:36830/solr/test_col_shard5_replica_n2/ update: org.apache.solr.client.solrj.request.UpdateRequest@6dc5b03f
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates$Worker.doRandomAtomicUpdate(TestStressCloudBlindAtomicUpdates.java:370)
at org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates$Worker.run(TestStressCloudBlindAtomicUpdates.java:399)
... 6 more
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Timeout waiting for all live and active
Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
at __randomizedtesting.SeedInfo.seed([C56A96109C8138C1:4D3EA9CA327D5539]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 8728 lines...]
[junit4] Suite: org.apache.lucene.search.TestInetAddressRangeQueries
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestInetAddressRangeQueries -Dtests.method=testRandomBig -Dtests.seed=A03631D71003B06D -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=fr-CH -Dtests.timezone=Africa/Cairo -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 1104s J1 | TestInetAddressRangeQueries.testRandomBig <<<
[junit4] > Throwable #1: java.lang.OutOfMemoryError: Java heap space
[junit4] > at __randomizedtesting.SeedInfo.seed([A03631D71003B06D:27614C58815ACCED]:0)
[junit4] > at org.apache.lucene.util.bkd.HeapPointWriter.writePackedValue(HeapPointWriter.java:107)
[junit4] > at org.apache.lucene.util.bkd.HeapPointWriter.append(HeapPointWriter.java:128)
[junit4] > at org.apache.lucene.util.bkd.PointReader.split(PointReader.java:68)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.build(BKDWriter.java:1791)
[junit4] > at org.apache.lucene.util.bkd.BKDWriter.finish(BKDWriter.java:1008)
[junit4] > at org.apache.lucene.index.RandomCodec$1$1.writeField(RandomCodec.java:141)
[junit4] > at org.apache.lucene.codecs.asserting.AssertingPointsFormat$AssertingPointsWriter.writeField(AssertingPointsFormat.java:137)
[junit4] > at org.apache.lucene.index.PointValuesWriter.flush(PointValuesWriter.java:183)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.writePoints(DefaultIndexingChain.java:203)
[junit4] > at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:138)
[junit4] > at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:451)
[junit4] > at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:542)
[junit4] > at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:399)
[junit4] > at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:502)
[junit4] > at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1717)
[junit4] > at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1462)
[junit4] > at org.apache.lucene.search.BaseRangeFieldQueryTestCase.verify(BaseRangeFieldQueryTestCase.java:190)
[junit4] > at org.apache.lucene.search.BaseRangeFieldQueryTestCase.doTestRandom(BaseRangeFieldQueryTestCase.java:160)
[junit4] > at org.apache.lucene.search.BaseRangeFieldQueryTestCase.testRandomBig(BaseRangeFieldQueryTestCase.java:75)
[junit4] > at org.apache.lucene.search.TestInetAddressRangeQueries.testRandomBig(TestInetAddressRangeQueries.java:81)
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/misc/test/J1/temp/lucene.search.TestInetAddressRangeQueries_A03631D71003B06D-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Direct)}, docValues:{ipRangeField=DocValuesFormat(name=Memory), id=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1019, maxMBSortInHeap=6.780535571636752, sim=RandomSimilarity(queryNorm=true): {}, locale=fr-CH, timezone=Africa/Cairo
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=252103960,total=371195904
[junit4] 2> NOTE: All tests run in this JVM: [TestInetAddressRangeQueries]
[junit4] Completed [12/13 (1!)] on J1 in 1160.35s, 4 tests, 1 error <<< FAILURES!
[...truncated 4 lines...]
[junit4] JVM J1: stdout was not empty, see: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/misc/test/temp/junit4-J1-20170905_222219_749533904013510541752.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/heapdumps/java_pid25144.hprof ...
[junit4] Heap dump file created [592704075 bytes in 10.654 secs]
[junit4] <<< JVM J1: EOF ----
[...truncated 1561 lines...]
[junit4] Suite: org.apache.lucene.spatial3d.TestGeo3DPoint
[junit4] 2> Zá? 05, 2017 8:08:24 ODP. com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2> WARNING: Suite execution timed out: org.apache.lucene.spatial3d.TestGeo3DPoint
[junit4] 2> 1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[16701B8549ADDA26], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> at org.apache.lucene.spatial3d.geom.Plane.recordBounds(Plane.java:1410)
[junit4] 2> at org.apache.lucene.spatial3d.geom.XYZBounds.addPlane(XYZBounds.java:161)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.getBounds(GeoComplexPolygon.java:380)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
[junit4] 2> 3) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2> at java.lang.Object.wait(Native Method)
[junit4] 2> at java.lang.Thread.join(Thread.java:1252)
[junit4] 2> at java.lang.Thread.join(Thread.java:1326)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:608)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:457)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
[junit4] 2> 4) Thread[id=11, name=SUITE-TestGeo3DPoint-seed#[16701B8549ADDA26], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> at java.lang.Thread.getStackTrace(Thread.java:1559)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
[junit4] 2> at java.security.AccessController.doPrivileged(Native Method)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:678)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:598)
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestGeo3DPoint -Dtests.method=testRandomBig -Dtests.seed=16701B8549ADDA26 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=cs -Dtests.timezone=America/Panama -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 7183s J1 | TestGeo3DPoint.testRandomBig <<<
[junit4] > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
[junit4] > at __randomizedtesting.SeedInfo.seed([16701B8549ADDA26]:0)
[junit4] 2> Zá? 05, 2017 8:08:24 ODP. com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> Zá? 05, 2017 8:08:44 ODP. com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.spatial3d.TestGeo3DPoint:
[junit4] 2> 1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[16701B8549ADDA26], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoComplexPolygon$DualCrossingEdgeIterator.<init>(GeoComplexPolygon.java:908)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.isWithin(GeoComplexPolygon.java:326)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoBaseMembershipShape.isWithin(GeoBaseMembershipShape.java:36)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoBaseShape.getBounds(GeoBaseShape.java:39)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.getBounds(GeoComplexPolygon.java:375)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Zá? 05, 2017 8:08:44 ODP. com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[16701B8549ADDA26], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> Zá? 05, 2017 8:08:47 ODP. com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> SEVERE: There are still zombie threads that couldn't be terminated:
[junit4] 2> 1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[16701B8549ADDA26], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> at org.apache.lucene.spatial3d.geom.Plane.findIntersections(Plane.java:650)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.isWithin(GeoComplexPolygon.java:164)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoBaseMembershipShape.isWithin(GeoBaseMembershipShape.java:36)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoBaseShape.getBounds(GeoBaseShape.java:52)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoComplexPolygon.getBounds(GeoComplexPolygon.java:375)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Memory)}, docValues:{id=DocValuesFormat(name=Memory), point=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1595, maxMBSortInHeap=7.688325144528704, sim=RandomSimilarity(queryNorm=false): {}, locale=cs, timezone=America/Panama
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=144690264,total=532152320
[junit4] 2> NOTE: All tests run in this JVM: [TestGeo3DPoint]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestGeo3DPoint -Dtests.seed=16701B8549ADDA26 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=cs -Dtests.timezone=America/Panama -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 0.00s J1 | TestGeo3DPoint (suite) <<<
[junit4] > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
[junit4] > at __randomizedtesting.SeedInfo.seed([16701B8549ADDA26]:0)
[junit4] Completed [10/11 (1!)] on J1 in 7224.92s, 6 tests, 2 errors <<< FAILURES!
[...truncated 1694 lines...]
[junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/init-core-data-001
[junit4] 2> 84469 WARN (SUITE-HttpPartitionTest-seed#[C56A96109C8138C1]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 84485 INFO (SUITE-HttpPartitionTest-seed#[C56A96109C8138C1]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 84489 INFO (SUITE-HttpPartitionTest-seed#[C56A96109C8138C1]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 84490 INFO (SUITE-HttpPartitionTest-seed#[C56A96109C8138C1]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /ggth/x
[junit4] 2> 84681 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 84725 INFO (Thread-32) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 84726 INFO (Thread-32) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 84789 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.ZkTestServer start zk server on port:35247
[junit4] 2> 84892 ERROR (Thread-32) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 85279 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 85281 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 85311 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 85312 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 85312 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 85313 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 85326 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 85327 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 85328 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 85329 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 85330 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 85411 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 88002 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 88022 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@199fcca0{/ggth/x,null,AVAILABLE}
[junit4] 2> 88033 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6add72d3{HTTP/1.1,[http/1.1]}{127.0.0.1:37569}
[junit4] 2> 88033 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.e.j.s.Server Started @100318ms
[junit4] 2> 88034 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/tempDir-001/control/data, hostContext=/ggth/x, hostPort=36295, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/control-001/cores}
[junit4] 2> 88034 ERROR (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 88034 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 88034 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 88034 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 88034 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-06T01:19:33.081Z
[junit4] 2> 88051 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 88051 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/control-001/solr.xml
[junit4] 2> 88120 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 88157 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35247/solr
[junit4] 2> 88611 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 88612 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:36295_ggth%2Fx
[junit4] 2> 88629 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.Overseer Overseer (id=98609448209481732-127.0.0.1:36295_ggth%2Fx-n_0000000000) starting
[junit4] 2> 88927 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36295_ggth%2Fx
[junit4] 2> 88929 INFO (zkCallback-27-thread-1-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 89748 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
[junit4] 2> 89881 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
[junit4] 2> 89943 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
[junit4] 2> 89945 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/control-001/cores
[junit4] 2> 90565 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 90565 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35247/solr ready
[junit4] 2> 90610 INFO (SocketProxy-Acceptor-36295) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=36930,localport=36295], receiveBufferSize:531000
[junit4] 2> 90739 INFO (SocketProxy-Acceptor-36295) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37569,localport=51450], receiveBufferSize=530904
[junit4] 2> 90810 INFO (qtp943224478-144) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:36295_ggth%252Fx&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 90875 INFO (OverseerThreadFactory-28-thread-1-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 91178 INFO (SocketProxy-Acceptor-36295) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=36934,localport=36295], receiveBufferSize:531000
[junit4] 2> 91227 INFO (SocketProxy-Acceptor-36295) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37569,localport=51454], receiveBufferSize=530904
[junit4] 2> 91232 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 91233 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 91419 INFO (zkCallback-27-thread-1-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 92529 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 93013 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 94663 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 94961 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 94962 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
[junit4] 2> 94962 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 94962 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 95037 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=42.5146484375, floorSegmentMB=0.98046875, forceMergeDeletesPctAllowed=29.48434745238268, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19348825750878085
[junit4] 2> 95198 WARN (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 95856 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 95872 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 95890 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 95904 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 95907 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@6b848d07
[junit4] 2> 95921 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1b196792[control_collection_shard1_replica_n1] main]
[junit4] 2> 95935 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 95935 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 95936 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 95983 INFO (searcherExecutor-31-thread-1-processing-n:127.0.0.1:36295_ggth%2Fx x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@1b196792[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 95983 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577751183129313280
[junit4] 2> 96110 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 96111 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 96111 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:36295/ggth/x/control_collection_shard1_replica_n1/
[junit4] 2> 96111 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 96111 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:36295/ggth/x/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 96111 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 96126 INFO (zkCallback-27-thread-1-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 96142 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:36295/ggth/x/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 96271 INFO (zkCallback-27-thread-1-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 96305 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 96369 INFO (qtp943224478-141) [n:127.0.0.1:36295_ggth%2Fx c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=5137
[junit4] 2> 96386 INFO (qtp943224478-144) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 96482 INFO (zkCallback-27-thread-1-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 96904 INFO (OverseerCollectionConfigSetProcessor-98609448209481732-127.0.0.1:36295_ggth%2Fx-n_0000000000) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 97395 INFO (qtp943224478-144) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:36295_ggth%252Fx&wt=javabin&version=2} status=0 QTime=6585
[junit4] 2> 97633 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 97633 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35247/solr ready
[junit4] 2> 97635 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 97650 INFO (SocketProxy-Acceptor-36295) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=37010,localport=36295], receiveBufferSize:531000
[junit4] 2> 97749 INFO (SocketProxy-Acceptor-36295) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37569,localport=51530], receiveBufferSize=530904
[junit4] 2> 97751 INFO (qtp943224478-145) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 97769 INFO (OverseerThreadFactory-28-thread-2-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 97849 WARN (OverseerThreadFactory-28-thread-2-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 98290 INFO (qtp943224478-145) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 98290 INFO (qtp943224478-145) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=538
[junit4] 2> 99063 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/shard-1-001 of type NRT
[junit4] 2> 99081 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 99194 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1fa1eb49{/ggth/x,null,AVAILABLE}
[junit4] 2> 99195 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6359f9fb{HTTP/1.1,[http/1.1]}{127.0.0.1:34330}
[junit4] 2> 99195 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.e.j.s.Server Started @111479ms
[junit4] 2> 99195 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/ggth/x, hostPort=39622, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/shard-1-001/cores}
[junit4] 2> 99195 ERROR (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 99196 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 99196 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 99196 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 99196 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-06T01:19:44.243Z
[junit4] 2> 99512 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 99512 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/shard-1-001/solr.xml
[junit4] 2> 99577 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 99641 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35247/solr
[junit4] 2> 99889 INFO (OverseerCollectionConfigSetProcessor-98609448209481732-127.0.0.1:36295_ggth%2Fx-n_0000000000) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 100014 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:39622_ggth%2Fx ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 100016 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:39622_ggth%2Fx ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 100017 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:39622_ggth%2Fx ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39622_ggth%2Fx
[junit4] 2> 100066 INFO (zkCallback-27-thread-2-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 100120 INFO (zkCallback-39-thread-1-processing-n:127.0.0.1:39622_ggth%2Fx) [n:127.0.0.1:39622_ggth%2Fx ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 100120 INFO (zkCallback-34-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 101284 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:39622_ggth%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
[junit4] 2> 101545 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:39622_ggth%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
[junit4] 2> 101545 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:39622_ggth%2Fx ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
[junit4] 2> 101565 INFO (TEST-HttpPartitionTest.test-seed#[C56A96109C8138C1]) [n:127.0.0.1:39622_ggth%2Fx ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/shard-1-001/cores
[junit4] 2> 102031 INFO (SocketProxy-Acceptor-39622) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54794,localport=39622], receiveBufferSize:531000
[junit4] 2> 102082 INFO (SocketProxy-Acceptor-39622) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=34330,localport=38058], receiveBufferSize=530904
[junit4] 2> 102119 INFO (qtp1548631192-192) [n:127.0.0.1:39622_ggth%2Fx ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:39622_ggth%252Fx&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 102292 INFO (OverseerThreadFactory-28-thread-3-processing-n:127.0.0.1:36295_ggth%2Fx) [n:127.0.0.1:36295_ggth%2Fx ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:39622_ggth%2Fx for creating new replica
[junit4] 2> 102298 INFO (SocketProxy-Acceptor-39622) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54826,localport=39622], receiveBufferSize:531000
[junit4] 2> 102340 INFO (SocketProxy-Acceptor-39622) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=34330,localport=38090], receiveBufferSize=530904
[junit4] 2> 102357 INFO (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 102358 INFO (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 103750 INFO (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 104105 INFO (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema [collection1_shard2_replica_n0] Schema name=test
[junit4] 2> 105917 INFO (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 106253 INFO (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n0' using configuration from collection collection1, trusted=true
[junit4] 2> 106253 INFO (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n0' (registry 'solr.core.collection1.shard2.replica_n0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@24f4bc86
[junit4] 2> 106253 INFO (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 106253 INFO (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [[collection1_shard2_replica_n0] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/shard-1-001/cores/collection1_shard2_replica_n0], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest_C56A96109C8138C1-001/shard-1-001/cores/collection1_shard2_replica_n0/data/]
[junit4] 2> 106300 INFO (qtp1548631192-190) [n:127.0.0.1:39622_ggth%2Fx c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=42.5146484375, floorSegmentMB=0.98046875, forceMergeDeletesPctAllowed=29.48434745238268, segmentsPerTier=46.0, maxCFS
[...truncated too long message...]
2> 833443 INFO (coreCloseExecutor-149-thread-1) [n:127.0.0.1:36393_solr c:test_col s:shard2 r:core_node3 x:test_col_shard2_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2df862d0 commitCommandVersion:0
[junit4] 2> 833454 INFO (coreCloseExecutor-148-thread-1) [n:127.0.0.1:45827_solr c:test_col s:shard3 r:core_node6 x:test_col_shard3_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.test_col.shard3.leader, tag=1240110536
[junit4] 2> 833454 INFO (coreCloseExecutor-148-thread-1) [n:127.0.0.1:45827_solr c:test_col s:shard3 r:core_node6 x:test_col_shard3_replica_n2] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter close.
[junit4] 2> 833454 INFO (coreCloseExecutor-148-thread-1) [n:127.0.0.1:45827_solr c:test_col s:shard3 r:core_node6 x:test_col_shard3_replica_n2] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@746f7d2a commitCommandVersion:0
[junit4] 2> 833744 INFO (zkCallback-50-thread-7-processing-n:127.0.0.1:36393_solr) [n:127.0.0.1:36393_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (7)
[junit4] 2> 833744 INFO (zkCallback-52-thread-9-processing-n:127.0.0.1:45766_solr) [n:127.0.0.1:45766_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (7)
[junit4] 2> 833744 INFO (zkCallback-53-thread-18-processing-n:127.0.0.1:42424_solr) [n:127.0.0.1:42424_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (7)
[junit4] 2> 833744 INFO (zkCallback-41-thread-9-processing-n:127.0.0.1:37931_solr) [n:127.0.0.1:37931_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (7)
[junit4] 2> 833755 WARN (zkCallback-49-thread-9-processing-n:127.0.0.1:34849_solr) [n:127.0.0.1:34849_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 833788 INFO (zkCallback-45-thread-9-processing-n:127.0.0.1:45746_solr) [n:127.0.0.1:45746_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (7)
[junit4] 2> 833788 INFO (zkCallback-47-thread-7-processing-n:127.0.0.1:36830_solr) [n:127.0.0.1:36830_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (7)
[junit4] 2> 833788 INFO (zkCallback-48-thread-7-processing-n:127.0.0.1:45827_solr) [n:127.0.0.1:45827_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (8) -> (7)
[junit4] 2> 833869 INFO (zkCallback-48-thread-7-processing-n:127.0.0.1:45827_solr) [n:127.0.0.1:45827_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (6)
[junit4] 2> 833871 INFO (jetty-closer-2-thread-7) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4afadeca{/solr,null,UNAVAILABLE}
[junit4] 2> 833922 INFO (zkCallback-50-thread-7-processing-n:127.0.0.1:36393_solr) [n:127.0.0.1:36393_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (6)
[junit4] 2> 833922 INFO (zkCallback-53-thread-18-processing-n:127.0.0.1:42424_solr) [n:127.0.0.1:42424_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (6)
[junit4] 2> 833922 INFO (zkCallback-41-thread-8-processing-n:127.0.0.1:37931_solr) [n:127.0.0.1:37931_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (6)
[junit4] 2> 833938 INFO (zkCallback-52-thread-9-processing-n:127.0.0.1:45766_solr) [n:127.0.0.1:45766_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (7) -> (6)
[junit4] 2> 833950 INFO (jetty-closer-2-thread-10) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@b0cb78b{/solr,null,UNAVAILABLE}
[junit4] 2> 834048 INFO (zkCallback-53-thread-18-processing-n:127.0.0.1:42424_solr) [n:127.0.0.1:42424_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (6) -> (1)
[junit4] 2> 834049 INFO (jetty-closer-2-thread-5) [ ] o.a.s.c.Overseer Overseer (id=98609442762326046-127.0.0.1:42424_solr-n_0000000000) closing
[junit4] 2> 834049 INFO (OverseerStateUpdate-98609442762326046-127.0.0.1:42424_solr-n_0000000000) [n:127.0.0.1:42424_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:42424_solr
[junit4] 2> 835358 WARN (zkCallback-45-thread-9-processing-n:127.0.0.1:45746_solr) [n:127.0.0.1:45746_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 835359 INFO (jetty-closer-2-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@795ae91e{/solr,null,UNAVAILABLE}
[junit4] 2> 835375 WARN (zkCallback-47-thread-7-processing-n:127.0.0.1:36830_solr) [n:127.0.0.1:36830_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 835375 INFO (jetty-closer-2-thread-6) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@9edc191{/solr,null,UNAVAILABLE}
[junit4] 2> 835423 WARN (zkCallback-48-thread-7-processing-n:127.0.0.1:45827_solr) [n:127.0.0.1:45827_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 835424 INFO (jetty-closer-2-thread-9) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@191879b5{/solr,null,UNAVAILABLE}
[junit4] 2> 835427 WARN (zkCallback-41-thread-8-processing-n:127.0.0.1:37931_solr) [n:127.0.0.1:37931_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 835427 WARN (zkCallback-50-thread-7-processing-n:127.0.0.1:36393_solr) [n:127.0.0.1:36393_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 835428 INFO (jetty-closer-2-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@31a2787b{/solr,null,UNAVAILABLE}
[junit4] 2> 835440 INFO (jetty-closer-2-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6cf8129{/solr,null,UNAVAILABLE}
[junit4] 2> 835671 WARN (zkCallback-53-thread-18-processing-n:127.0.0.1:42424_solr) [n:127.0.0.1:42424_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 835688 INFO (jetty-closer-2-thread-5) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@71b537ee{/solr,null,UNAVAILABLE}
[junit4] 2> 835750 ERROR (SUITE-TestStressCloudBlindAtomicUpdates-seed#[C56A96109C8138C1]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 835765 INFO (SUITE-TestStressCloudBlindAtomicUpdates-seed#[C56A96109C8138C1]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44912 44912
[junit4] 2> 836031 INFO (Thread-1) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44912 44912
[junit4] 2> 836187 WARN (Thread-1) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 11 /solr/aliases.json
[junit4] 2> 11 /solr/clusterprops.json
[junit4] 2> 10 /solr/configs/org.apache.solr.cloud.TestStressCloudBlindAtomicUpdates_config-set
[junit4] 2> 10 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 4229 /solr/collections/test_col/state.json
[junit4] 2> 11 /solr/clusterstate.json
[junit4] 2> 2 /solr/overseer_elect/election/98609442762326046-127.0.0.1:42424_solr-n_0000000000
[junit4] 2> 2 /solr/overseer_elect/election/98609442762326045-127.0.0.1:45766_solr-n_0000000003
[junit4] 2> 2 /solr/overseer_elect/election/98609442762326033-127.0.0.1:37931_solr-n_0000000005
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 11 /solr/live_nodes
[junit4] 2> 11 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressCloudBlindAtomicUpdates_C56A96109C8138C1-001
[junit4] 2> Sep 06, 2017 1:32:01 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=SimpleText, sim=RandomSimilarity(queryNorm=false): {}, locale=sv-SE, timezone=Europe/Zaporozhye
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=210055568,total=405274624
[junit4] 2> NOTE: All tests run in this JVM: [TestStressCloudBlindAtomicUpdates]
[junit4] Completed [25/729 (3!)] on J1 in 844.25s, 5 tests, 1 error <<< FAILURES!
[...truncated 10020 lines...]
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/build.xml:818: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/build.xml:762: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid25144.hprof
Total time: 766 minutes 58 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 37 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/37/
2 tests failed.
FAILED: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest.test
Error Message:
Mismatch in counts between replicas
Stack Trace:
java.lang.AssertionError: Mismatch in counts between replicas
at __randomizedtesting.SeedInfo.seed([574A0B0E76428B14:DF1E34D4D8BEE6EC]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.RecoveryZkTest.assertShardConsistency(RecoveryZkTest.java:143)
at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:126)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Timeout waiting for all live and active
Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
at __randomizedtesting.SeedInfo.seed([574A0B0E76428B14:DF1E34D4D8BEE6EC]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:317)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13347 lines...]
[junit4] Suite: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/init-core-data-001
[junit4] 2> 4860828 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 4860832 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
[junit4] 2> 4860833 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /hkse/wu
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 4860906 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 4860918 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 4860920 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 4860930 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_37308_hdfs____k6l935/webapp
[junit4] 2> 4861409 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37308
[junit4] 2> 4861496 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 4861497 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 4861508 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_59592_datanode____.rffhr/webapp
[junit4] 2> 4861977 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:59592
[junit4] 2> 4862105 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 4862106 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 4862129 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_60624_datanode____tav5yd/webapp
[junit4] 2> 4862293 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:35660) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 4862456 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x10edd65c3d3a3c: from storage DS-7cf99784-6d71-48a5-925a-db90f081c7df node DatanodeRegistration(127.0.0.1:53681, datanodeUuid=9acdf709-cfa9-4942-ae91-cffab6199131, infoPort=59557, infoSecurePort=0, ipcPort=37835, storageInfo=lv=-56;cid=testClusterID;nsid=318144730;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 4862456 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x10edd65c3d3a3c: from storage DS-f4f5b8bb-5f3a-4b4d-bddb-d56708fd76e5 node DatanodeRegistration(127.0.0.1:53681, datanodeUuid=9acdf709-cfa9-4942-ae91-cffab6199131, infoPort=59557, infoSecurePort=0, ipcPort=37835, storageInfo=lv=-56;cid=testClusterID;nsid=318144730;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 4862719 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[574A0B0E76428B14]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:60624
[junit4] 2> 4862853 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:35660) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 4862862 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x10edd6745bd504: from storage DS-3bb22fb1-1c81-46dd-9ec8-6bbd7a2cf585 node DatanodeRegistration(127.0.0.1:43710, datanodeUuid=ab5672b4-645f-446e-97b3-69f148aadf16, infoPort=50230, infoSecurePort=0, ipcPort=51109, storageInfo=lv=-56;cid=testClusterID;nsid=318144730;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 4862862 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x10edd6745bd504: from storage DS-febeccb2-bad1-408e-b719-39e14e3af424 node DatanodeRegistration(127.0.0.1:43710, datanodeUuid=ab5672b4-645f-446e-97b3-69f148aadf16, infoPort=50230, infoSecurePort=0, ipcPort=51109, storageInfo=lv=-56;cid=testClusterID;nsid=318144730;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 4863792 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 4863792 INFO (Thread-54437) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 4863792 INFO (Thread-54437) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 4863793 ERROR (Thread-54437) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 4863898 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.ZkTestServer start zk server on port:56152
[junit4] 2> 4863918 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 4863919 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 4863920 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 4863921 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 4863921 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 4863926 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 4863930 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 4863930 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 4863931 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 4863932 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 4863932 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 4863933 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 4864024 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 4864025 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3c14ec1c{/hkse/wu,null,AVAILABLE}
[junit4] 2> 4864025 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3b74f913{HTTP/1.1,[http/1.1]}{127.0.0.1:32804}
[junit4] 2> 4864025 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.Server Started @4868762ms
[junit4] 2> 4864025 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/hkse/wu, hostPort=32804, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/control-001/cores}
[junit4] 2> 4864025 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4864025 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 4864026 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4864026 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 4864026 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-04T08:19:38.363Z
[junit4] 2> 4864034 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 4864034 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/control-001/solr.xml
[junit4] 2> 4864038 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 4864041 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56152/solr
[junit4] 2> 4864090 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 4864090 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:32804_hkse%2Fwu
[junit4] 2> 4864091 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.Overseer Overseer (id=98599775641927684-127.0.0.1:32804_hkse%2Fwu-n_0000000000) starting
[junit4] 2> 4864096 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32804_hkse%2Fwu
[junit4] 2> 4864097 INFO (zkCallback-26581-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4864374 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
[junit4] 2> 4864383 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
[junit4] 2> 4864383 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
[junit4] 2> 4864384 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/control-001/cores
[junit4] 2> 4864433 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4864434 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56152/solr ready
[junit4] 2> 4864439 INFO (qtp929749809-161388) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:32804_hkse%252Fwu&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4864450 INFO (OverseerThreadFactory-9664-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 4864585 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 4864585 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 4864688 INFO (zkCallback-26581-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4865608 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 4865650 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 4865714 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 4865724 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 4865725 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
[junit4] 2> 4865725 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:35660/solr_hdfs_home
[junit4] 2> 4865725 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 4865725 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 4865725 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://localhost:35660/solr_hdfs_home/control_collection/core_node1/data/]
[junit4] 2> 4865726 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/control_collection/core_node1/data/snapshot_metadata
[junit4] 2> 4865733 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 4865733 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
[junit4] 2> 4865742 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 4865743 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/control_collection/core_node1/data
[junit4] 2> 4865760 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/control_collection/core_node1/data/index
[junit4] 2> 4865765 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 4865765 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
[junit4] 2> 4865768 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 4865768 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=3, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=17.220703125, floorSegmentMB=0.423828125, forceMergeDeletesPctAllowed=4.465370431227933, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6594933322814993
[junit4] 2> 4865787 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:43710 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-f4f5b8bb-5f3a-4b4d-bddb-d56708fd76e5:NORMAL:127.0.0.1:53681|RBW], ReplicaUC[[DISK]DS-3bb22fb1-1c81-46dd-9ec8-6bbd7a2cf585:NORMAL:127.0.0.1:43710|RBW]]} size 0
[junit4] 2> 4865789 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:53681 is added to blk_1073741825_1001 size 72
[junit4] 2> 4865799 WARN (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 4865961 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 4865961 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 4865961 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 4865981 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 4865981 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 4865982 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=2, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8887442367528195]
[junit4] 2> 4865999 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6cc5391e[control_collection_shard1_replica_n1] main]
[junit4] 2> 4866000 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 4866000 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 4866000 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 4866002 INFO (searcherExecutor-9667-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@6cc5391e[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 4866002 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577596412587147264
[junit4] 2> 4866010 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 4866010 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 4866010 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:32804/hkse/wu/control_collection_shard1_replica_n1/
[junit4] 2> 4866010 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 4866010 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:32804/hkse/wu/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 4866010 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 4866012 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:32804/hkse/wu/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 4866114 INFO (zkCallback-26581-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4866168 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 4866169 INFO (qtp929749809-161386) [n:127.0.0.1:32804_hkse%2Fwu c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1584
[junit4] 2> 4866177 INFO (qtp929749809-161388) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 4866278 INFO (zkCallback-26581-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4866458 INFO (OverseerCollectionConfigSetProcessor-98599775641927684-127.0.0.1:32804_hkse%2Fwu-n_0000000000) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 4867178 INFO (qtp929749809-161388) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:32804_hkse%252Fwu&wt=javabin&version=2} status=0 QTime=2739
[junit4] 2> 4867184 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4867185 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56152/solr ready
[junit4] 2> 4867185 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 4867186 INFO (qtp929749809-161385) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4867187 INFO (OverseerThreadFactory-9664-thread-2-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 4867188 WARN (OverseerThreadFactory-9664-thread-2-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 4867394 INFO (qtp929749809-161385) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 4867402 INFO (qtp929749809-161385) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=215
[junit4] 2> 4867498 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-1-001 of type NRT
[junit4] 2> 4867498 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 4867499 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7b8f407a{/hkse/wu,null,AVAILABLE}
[junit4] 2> 4867500 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3ce27d3f{HTTP/1.1,[http/1.1]}{127.0.0.1:42830}
[junit4] 2> 4867500 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.Server Started @4872237ms
[junit4] 2> 4867500 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/hkse/wu, hostPort=42830, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-1-001/cores}
[junit4] 2> 4867500 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4867500 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 4867500 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4867500 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 4867500 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-04T08:19:41.837Z
[junit4] 2> 4867527 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 4867527 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-1-001/solr.xml
[junit4] 2> 4867531 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 4867534 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56152/solr
[junit4] 2> 4867543 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4867544 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 4867545 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42830_hkse%2Fwu
[junit4] 2> 4867546 INFO (zkCallback-26581-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 4867553 INFO (zkCallback-26588-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 4867579 INFO (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 4867624 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
[junit4] 2> 4867634 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
[junit4] 2> 4867634 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
[junit4] 2> 4867636 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-1-001/cores
[junit4] 2> 4867658 INFO (qtp1753547614-161439) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:42830_hkse%252Fwu&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4867659 INFO (OverseerCollectionConfigSetProcessor-98599775641927684-127.0.0.1:32804_hkse%2Fwu-n_0000000000) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 4867660 INFO (OverseerThreadFactory-9664-thread-3-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:42830_hkse%2Fwu for creating new replica
[junit4] 2> 4867661 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 4867661 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 4867764 INFO (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4868681 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 4868692 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema [collection1_shard2_replica_n0] Schema name=test
[junit4] 2> 4868768 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 4868798 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n0' using configuration from collection collection1, trusted=true
[junit4] 2> 4868798 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n0' (registry 'solr.core.collection1.shard2.replica_n0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
[junit4] 2> 4868798 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:35660/solr_hdfs_home
[junit4] 2> 4868798 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 4868798 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 4868798 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [[collection1_shard2_replica_n0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-1-001/cores/collection1_shard2_replica_n0], dataDir=[hdfs://localhost:35660/solr_hdfs_home/collection1/core_node1/data/]
[junit4] 2> 4868800 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/collection1/core_node1/data/snapshot_metadata
[junit4] 2> 4868824 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 4868824 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
[junit4] 2> 4868845 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 4868846 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/collection1/core_node1/data
[junit4] 2> 4868875 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/collection1/core_node1/data/index
[junit4] 2> 4868884 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 4868884 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
[junit4] 2> 4868896 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 4868897 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=3, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=17.220703125, floorSegmentMB=0.423828125, forceMergeDeletesPctAllowed=4.465370431227933, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6594933322814993
[junit4] 2> 4868944 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:53681 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-febeccb2-bad1-408e-b719-39e14e3af424:NORMAL:127.0.0.1:43710|RBW], ReplicaUC[[DISK]DS-7cf99784-6d71-48a5-925a-db90f081c7df:NORMAL:127.0.0.1:53681|FINALIZED]]} size 0
[junit4] 2> 4868946 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:43710 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-febeccb2-bad1-408e-b719-39e14e3af424:NORMAL:127.0.0.1:43710|RBW], ReplicaUC[[DISK]DS-7cf99784-6d71-48a5-925a-db90f081c7df:NORMAL:127.0.0.1:53681|FINALIZED]]} size 0
[junit4] 2> 4868979 WARN (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 4869017 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 4869017 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 4869017 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 4869025 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 4869026 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 4869035 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=2, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8887442367528195]
[junit4] 2> 4869067 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.SolrIndexSearcher Opening [Searcher@4552098b[collection1_shard2_replica_n0] main]
[junit4] 2> 4869068 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 4869068 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 4869068 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 4869079 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577596415813615616
[junit4] 2> 4869081 INFO (searcherExecutor-9678-thread-1-processing-n:127.0.0.1:42830_hkse%2Fwu x:collection1_shard2_replica_n0 s:shard2 c:collection1) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SolrCore [collection1_shard2_replica_n0] Registered new searcher Searcher@4552098b[collection1_shard2_replica_n0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 4869084 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 4869084 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 4869084 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:42830/hkse/wu/collection1_shard2_replica_n0/
[junit4] 2> 4869084 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 4869084 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.SyncStrategy http://127.0.0.1:42830/hkse/wu/collection1_shard2_replica_n0/ has no replicas
[junit4] 2> 4869084 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 4869086 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:42830/hkse/wu/collection1_shard2_replica_n0/ shard2
[junit4] 2> 4869200 INFO (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4869247 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 4869248 INFO (qtp1753547614-161437) [n:127.0.0.1:42830_hkse%2Fwu c:collection1 s:shard2 x:collection1_shard2_replica_n0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n0&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1587
[junit4] 2> 4869250 INFO (qtp1753547614-161439) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:42830_hkse%252Fwu&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1592
[junit4] 2> 4869350 INFO (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4869443 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-2-001 of type NRT
[junit4] 2> 4869443 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 4869472 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4f722e82{/hkse/wu,null,AVAILABLE}
[junit4] 2> 4869472 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7244f219{HTTP/1.1,[http/1.1]}{127.0.0.1:41450}
[junit4] 2> 4869472 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.Server Started @4874209ms
[junit4] 2> 4869472 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/hkse/wu, hostPort=41450, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-2-001/cores}
[junit4] 2> 4869473 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4869473 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 4869473 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4869474 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 4869474 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-04T08:19:43.811Z
[junit4] 2> 4869476 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 4869476 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-2-001/solr.xml
[junit4] 2> 4869480 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 4869498 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56152/solr
[junit4] 2> 4869524 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:41450_hkse%2Fwu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 4869526 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:41450_hkse%2Fwu ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 4869527 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:41450_hkse%2Fwu ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41450_hkse%2Fwu
[junit4] 2> 4869528 INFO (zkCallback-26581-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 4869528 INFO (zkCallback-26588-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 4869538 INFO (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 4869558 INFO (zkCallback-26599-thread-1-processing-n:127.0.0.1:41450_hkse%2Fwu) [n:127.0.0.1:41450_hkse%2Fwu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 4869656 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:41450_hkse%2Fwu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
[junit4] 2> 4869660 INFO (OverseerCollectionConfigSetProcessor-98599775641927684-127.0.0.1:32804_hkse%2Fwu-n_0000000000) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 4869667 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:41450_hkse%2Fwu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
[junit4] 2> 4869667 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:41450_hkse%2Fwu ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
[junit4] 2> 4869672 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:41450_hkse%2Fwu ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-2-001/cores
[junit4] 2> 4869706 INFO (qtp1753547614-161440) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41450_hkse%252Fwu&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4869715 INFO (OverseerThreadFactory-9664-thread-4-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:41450_hkse%2Fwu for creating new replica
[junit4] 2> 4869717 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 4869717 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 4869821 INFO (zkCallback-26599-thread-1-processing-n:127.0.0.1:41450_hkse%2Fwu) [n:127.0.0.1:41450_hkse%2Fwu ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4869821 INFO (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4870733 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 4870750 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.s.IndexSchema [collection1_shard1_replica_n0] Schema name=test
[junit4] 2> 4871055 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 4871066 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n0' using configuration from collection collection1, trusted=true
[junit4] 2> 4871066 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n0' (registry 'solr.core.collection1.shard1.replica_n0') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b
[junit4] 2> 4871066 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:35660/solr_hdfs_home
[junit4] 2> 4871067 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 4871067 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 4871067 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SolrCore [[collection1_shard1_replica_n0] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-2-001/cores/collection1_shard1_replica_n0], dataDir=[hdfs://localhost:35660/solr_hdfs_home/collection1/core_node2/data/]
[junit4] 2> 4871068 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/collection1/core_node2/data/snapshot_metadata
[junit4] 2> 4871074 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 4871074 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
[junit4] 2> 4871081 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 4871082 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/collection1/core_node2/data
[junit4] 2> 4871107 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35660/solr_hdfs_home/collection1/core_node2/data/index
[junit4] 2> 4871112 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 4871112 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [4194304] will allocate [1] slabs and use ~[4194304] bytes
[junit4] 2> 4871118 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 4871119 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=3, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=17.220703125, floorSegmentMB=0.423828125, forceMergeDeletesPctAllowed=4.465370431227933, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6594933322814993
[junit4] 2> 4871135 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:43710 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-7cf99784-6d71-48a5-925a-db90f081c7df:NORMAL:127.0.0.1:53681|RBW], ReplicaUC[[DISK]DS-3bb22fb1-1c81-46dd-9ec8-6bbd7a2cf585:NORMAL:127.0.0.1:43710|RBW]]} size 0
[junit4] 2> 4871135 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:53681 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-3bb22fb1-1c81-46dd-9ec8-6bbd7a2cf585:NORMAL:127.0.0.1:43710|RBW], ReplicaUC[[DISK]DS-f4f5b8bb-5f3a-4b4d-bddb-d56708fd76e5:NORMAL:127.0.0.1:53681|FINALIZED]]} size 0
[junit4] 2> 4871140 WARN (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 4871186 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 4871186 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 4871186 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=2
[junit4] 2> 4871195 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 4871195 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 4871196 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=2, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8887442367528195]
[junit4] 2> 4871199 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.s.SolrIndexSearcher Opening [Searcher@200818eb[collection1_shard1_replica_n0] main]
[junit4] 2> 4871200 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 4871201 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 4871222 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 4871223 INFO (searcherExecutor-9689-thread-1-processing-n:127.0.0.1:41450_hkse%2Fwu x:collection1_shard1_replica_n0 s:shard1 c:collection1) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SolrCore [collection1_shard1_replica_n0] Registered new searcher Searcher@200818eb[collection1_shard1_replica_n0] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 4871224 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577596418062811136
[junit4] 2> 4871234 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 4871234 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 4871234 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:41450/hkse/wu/collection1_shard1_replica_n0/
[junit4] 2> 4871234 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 4871234 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.SyncStrategy http://127.0.0.1:41450/hkse/wu/collection1_shard1_replica_n0/ has no replicas
[junit4] 2> 4871234 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 4871237 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:41450/hkse/wu/collection1_shard1_replica_n0/ shard1
[junit4] 2> 4871338 INFO (zkCallback-26599-thread-1-processing-n:127.0.0.1:41450_hkse%2Fwu) [n:127.0.0.1:41450_hkse%2Fwu ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4871338 INFO (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4871388 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 4871389 INFO (qtp557264569-161477) [n:127.0.0.1:41450_hkse%2Fwu c:collection1 s:shard1 x:collection1_shard1_replica_n0] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n0&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1672
[junit4] 2> 4871399 INFO (qtp1753547614-161440) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:41450_hkse%252Fwu&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1693
[junit4] 2> 4871491 INFO (zkCallback-26599-thread-1-processing-n:127.0.0.1:41450_hkse%2Fwu) [n:127.0.0.1:41450_hkse%2Fwu ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4871491 INFO (zkCallback-26593-thread-2-processing-n:127.0.0.1:42830_hkse%2Fwu) [n:127.0.0.1:42830_hkse%2Fwu ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4871514 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-3-001 of type NRT
[junit4] 2> 4871514 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 4871515 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6227168e{/hkse/wu,null,AVAILABLE}
[junit4] 2> 4871515 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2f960b78{HTTP/1.1,[http/1.1]}{127.0.0.1:43591}
[junit4] 2> 4871515 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.e.j.s.Server Started @4876253ms
[junit4] 2> 4871515 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solrconfig=solrconfig.xml, hostContext=/hkse/wu, hostPort=43591, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-3-001/cores}
[junit4] 2> 4871516 ERROR (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4871525 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 4871525 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4871525 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 4871525 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-04T08:19:45.862Z
[junit4] 2> 4871535 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 4871535 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_574A0B0E76428B14-001/shard-3-001/solr.xml
[junit4] 2> 4871541 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb3a81b, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 4871544 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56152/solr
[junit4] 2> 4871549 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:43591_hkse%2Fwu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 4871550 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:43591_hkse%2Fwu ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 4871552 INFO (TEST-SharedFSAutoReplicaFailoverTest.test-seed#[574A0B0E76428B14]) [n:127.0.0.1:43591_hkse%2Fwu ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43591_hkse%2Fwu
[junit4] 2> 4871553 INFO (zkCallback-26588-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 4871553 INFO (zkCallback-26599-thread-1-processing-n:127.0.0.1:41450_hkse%2Fwu) [n:127.0.0.1:41450_hkse%2Fwu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 4871563 INFO (zkCallback-26581-thread-1-processing-n:127.0.0.1:32804_hkse%2Fwu) [n:127.0.0.1:32804_hkse%2Fwu ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 4871563 INFO (zkCallback-26593-thread-2-proc
[...truncated too long message...]
-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_574A0B0E76428B14-001/tempDir-002/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:34855) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-1068949724-10.41.0.5-1504513889705 (Datanode Uuid 59b2686d-2d56-4bf1-8314-8f210305eb97) service to localhost/127.0.0.1:34855
[junit4] 2> 5602691 WARN (SUITE-HdfsRecoveryZkTest-seed#[574A0B0E76428B14]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 5602735 INFO (SUITE-HdfsRecoveryZkTest-seed#[574A0B0E76428B14]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 5602736 WARN (296771641@qtp-118835101-0 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:54539) [ ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false. Rechecking.
[junit4] 2> 5602737 WARN (296771641@qtp-118835101-0 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:54539) [ ] o.a.h.h.HttpServer2 HttpServer Acceptor: isRunning is false
[junit4] 2> 5602739 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_574A0B0E76428B14-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_574A0B0E76428B14-001/tempDir-002/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:34855) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
[junit4] 2> 5602739 WARN (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_574A0B0E76428B14-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_574A0B0E76428B14-001/tempDir-002/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:34855) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-1068949724-10.41.0.5-1504513889705 (Datanode Uuid 02c98771-33f5-4d2b-b478-76e0e77e02dd) service to localhost/127.0.0.1:34855
[junit4] 2> 5602758 INFO (SUITE-HdfsRecoveryZkTest-seed#[574A0B0E76428B14]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 5602765 INFO (SUITE-HdfsRecoveryZkTest-seed#[574A0B0E76428B14]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39453 39453
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_574A0B0E76428B14-001
[junit4] 2> Sep 04, 2017 8:32:00 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {rnd_b=PostingsFormat(name=Memory), _version_=PostingsFormat(name=Direct), a_t=PostingsFormat(name=Direct), a_i=PostingsFormat(name=Memory), id=PostingsFormat(name=Memory)}, docValues:{_version_=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1274, maxMBSortInHeap=5.470026215518962, sim=RandomSimilarity(queryNorm=false): {}, locale=hr-HR, timezone=Mexico/BajaNorte
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=199049240,total=522715136
[junit4] 2> NOTE: All tests run in this JVM: [TestExclusionRuleCollectionAccess, CdcrVersionReplicationTest, AnalyticsMergeStrategyTest, HdfsBasicDistributedZkTest, TlogReplayBufferedWhileIndexingTest, SpellCheckComponentTest, DocValuesMultiTest, RestartWhileUpdatingTest, JsonLoaderTest, TestDFISimilarityFactory, TestBinaryResponseWriter, TestFiltering, TestCursorMarkWithoutUniqueKey, OpenCloseCoreStressTest, SolrJmxReporterCloudTest, BitVectorTest, DistributedQueryComponentCustomSortTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, CollectionsAPIDistributedZkTest, TestFastLRUCache, NoCacheHeaderTest, TestMiniSolrCloudClusterSSL, RequestLoggingTest, TestHashQParserPlugin, TestClusterProperties, TestCoreAdminApis, SchemaVersionSpecificBehaviorTest, TestXIncludeConfig, CdcrReplicationDistributedZkTest, UpdateRequestProcessorFactoryTest, TestSerializedLuceneMatchVersion, CoreSorterTest, ZkStateReaderTest, TestCloudDeleteByQuery, DistributedMLTComponentTest, ActionThrottleTest, TestCoreContainer, TestSmileRequest, TestShardHandlerFactory, SpatialRPTFieldTypeTest, TestSolrXml, SOLR749Test, TestQueryTypes, TestValueSourceCache, TestNumericRangeQuery32, TestSweetSpotSimilarityFactory, DirectSolrConnectionTest, TestMacros, TestSolrCloudWithDelegationTokens, CacheHeaderTest, TestDocBasedVersionConstraints, BasicFunctionalityTest, AlternateDirectoryTest, JavabinLoaderTest, DateRangeFieldTest, HdfsCollectionsAPIDistributedZkTest, TestSolrFieldCacheBean, SecurityConfHandlerTest, ClassificationUpdateProcessorTest, TestTlogReplica, TestJsonFacetRefinement, TestTrieFacet, ResourceLoaderTest, InfoHandlerTest, TestDistributedGrouping, TestFieldCacheReopen, SaslZkACLProviderTest, SystemInfoHandlerTest, ShardRoutingTest, TestStressRecovery, DistanceFunctionTest, TestStressVersions, ConfigSetsAPITest, TestPayloadScoreQParserPlugin, TestFieldTypeCollectionResource, TestDynamicLoading, CollectionsAPIAsyncDistributedZkTest, AutoScalingHandlerTest, TestAuthorizationFramework, LoggingHandlerTest, TestUpdate, SpatialFilterTest, TestHttpShardHandlerFactory, TestPerFieldSimilarity, CheckHdfsIndexTest, TestSlowCompositeReaderWrapper, TestPostingsSolrHighlighter, TestCharFilters, DocValuesTest, TestSchemaResource, TestManagedResourceStorage, BlockDirectoryTest, FacetPivotSmallTest, TestSimpleTrackingShardHandler, DistribJoinFromCollectionTest, TestFieldSortValues, TestSQLHandlerNonCloud, MoveReplicaTest, BlockJoinFacetDistribTest, TestComplexPhraseLeadingWildcard, AddBlockUpdateTest, SoftAutoCommitTest, MetricUtilsTest, TestRTGBase, AsyncCallRequestStatusResponseTest, TestSegmentSorting, TestCloudPivotFacet, TestDynamicFieldResource, TestConfigSetsAPIExclusivity, TestAnalyzeInfixSuggestions, SolrCmdDistributorTest, ShufflingReplicaListTransformerTest, SampleTest, SolrCoreMetricManagerTest, DistributedFacetExistsSmallTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, SyncSliceTest, OverseerTest, FullSolrCloudDistribCmdsTest, ClusterStateUpdateTest, TestRandomFaceting, ZkSolrClientTest, TestZkChroot, TestRandomDVFaceting, ZkCLITest, ShardRoutingCustomTest, AutoCommitTest, TestFaceting, TestHashPartitioner, ZkControllerTest, TestRealTimeGet, TestStressReorder, TestJoin, TestReload, HardAutoCommitTest, DistributedTermsComponentTest, TestSolr4Spatial, SolrCoreTest, StatsComponentTest, BadIndexSchemaTest, ConvertedLegacyTest, TestFunctionQuery, DirectUpdateHandlerTest, TestBadConfig, HighlighterTest, SimplePostToolTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, TestExtendedDismaxParser, CoreAdminHandlerTest, SuggesterTSTTest, TestStressLucene, TestCSVLoader, PolyFieldTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, QueryEqualityTest, DirectUpdateHandlerOptimizeTest, DefaultValueUpdateProcessorTest, StatelessScriptUpdateProcessorFactoryTest, SortByFunctionTest, XsltUpdateRequestHandlerTest, LukeRequestHandlerTest, DisMaxRequestHandlerTest, TestWriterPerf, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, TestOmitPositions, FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest, DocumentBuilderTest, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, TestIndexingPerformance, MoreLikeThisHandlerTest, RequiredFieldsTest, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, SolrPluginUtilsTest, IndexSchemaTest, TestCollationField, ReturnFieldsTest, MBeansHandlerTest, QueryParsingTest, BinaryUpdateRequestHandlerTest, TestPartialUpdateDeduplication, PingRequestHandlerTest, BadComponentTest, TestSolrDeletionPolicy2, TestDocSet, NumericFieldsTest, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, EchoParamsTest, TestLMJelinekMercerSimilarityFactory, TimeZoneUtilsTest, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, PreAnalyzedFieldTest, PrimUtilsTest, DOMUtilTest, TestSolrJ, TestUtils, ZkNodePropsTest, UUIDFieldTest, CircularListTest, DistributedIntervalFacetingTest, CdcrRequestHandlerTest, CollectionsAPISolrJTest, DeleteStatusTest, DistributedQueueTest, DistributedVersionInfoTest, LeaderElectionContextKeyTest, LeaderFailureAfterFreshStartTest, LeaderInitiatedRecoveryOnCommitTest, MissingSegmentRecoveryTest, MoveReplicaHDFSTest, MoveReplicaHDFSUlogDirTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerModifyCollectionTest, OverseerStatusTest, PeerSyncReplicationTest, RemoteQueryErrorTest, SharedFSAutoReplicaFailoverUtilsTest, SolrCLIZkUtilsTest, SolrCloudExampleTest, TestAuthenticationFramework, TestCloudPseudoReturnFields, TestCloudRecovery, TestConfigSetsAPI, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestHdfsCloudBackupRestore, TestLeaderElectionWithEmptyReplica, TestLeaderElectionZkExpiry, TestLeaderInitiatedRecoveryThread, TestLockTree, TestMiniSolrCloudCluster, TestOnReconnectListenerSupport, TestPullReplica, TestRandomFlRTGCloud, TestStressInPlaceUpdates, TestPolicyCloud, HdfsChaosMonkeyNothingIsSafeTest, HdfsRecoveryZkTest]
[junit4] Completed [555/729 (2!)] on J1 in 41.48s, 1 test, 1 failure <<< FAILURES!
[...truncated 43900 lines...]
[JENKINS] Lucene-Solr-NightlyTests-7.0 - Build # 36 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.0/36/
5 tests failed.
FAILED: org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([23DB77BF5E4E778A]:0)
FAILED: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest.test
Error Message:
Stack Trace:
java.util.concurrent.TimeoutException
at __randomizedtesting.SeedInfo.seed([7FA8F1A6FFF1E22D:F7FCCE7C510D8FD5]:0)
at org.apache.solr.common.cloud.ZkStateReader.waitForState(ZkStateReader.java:1267)
at org.apache.solr.client.solrj.impl.CloudSolrClient.waitForState(CloudSolrClient.java:438)
at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:122)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster
Error Message:
Document mismatch on target after sync expected:<20000> but was:<10100>
Stack Trace:
java.lang.AssertionError: Document mismatch on target after sync expected:<20000> but was:<10100>
at __randomizedtesting.SeedInfo.seed([7FA8F1A6FFF1E22D:ABEDBAFF18A751D6]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.apache.solr.cloud.CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster(CdcrBootstrapTest.java:309)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.lucene.spatial3d.TestGeo3DPoint
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([23DB77BF5E4E778A]:0)
FAILED: org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test
Error Message:
Timeout waiting for all live and active
Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
at __randomizedtesting.SeedInfo.seed([7FA8F1A6FFF1E22D:F7FCCE7C510D8FD5]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 10149 lines...]
[junit4] Suite: org.apache.lucene.spatial3d.TestGeo3DPoint
[junit4] 2> ??? 01, 2017 5:51:40 ? com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
[junit4] 2> WARNING: Suite execution timed out: org.apache.lucene.spatial3d.TestGeo3DPoint
[junit4] 2> 1) Thread[id=11, name=SUITE-TestGeo3DPoint-seed#[23DB77BF5E4E778A], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> at java.lang.Thread.getStackTrace(Thread.java:1559)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:696)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:693)
[junit4] 2> at java.security.AccessController.doPrivileged(Native Method)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:693)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:709)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:689)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:65)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:415)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:678)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:598)
[junit4] 2> 2) Thread[id=1, name=main, state=WAITING, group=main]
[junit4] 2> at java.lang.Object.wait(Native Method)
[junit4] 2> at java.lang.Thread.join(Thread.java:1252)
[junit4] 2> at java.lang.Thread.join(Thread.java:1326)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:608)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:457)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
[junit4] 2> 3) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[23DB77BF5E4E778A], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> at org.apache.lucene.spatial3d.geom.Plane.addPoint(Plane.java:1953)
[junit4] 2> at org.apache.lucene.spatial3d.geom.Plane.recordBounds(Plane.java:1418)
[junit4] 2> at org.apache.lucene.spatial3d.geom.XYZBounds.addPlane(XYZBounds.java:161)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoConvexPolygon.getBounds(GeoConvexPolygon.java:423)
[junit4] 2> at org.apache.lucene.spatial3d.geom.GeoCompositeMembershipShape.getBounds(GeoCompositeMembershipShape.java:75)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:846)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 4) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestGeo3DPoint -Dtests.method=testRandomBig -Dtests.seed=23DB77BF5E4E778A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=ar-OM -Dtests.timezone=Pacific/Guam -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 7196s J1 | TestGeo3DPoint.testRandomBig <<<
[junit4] > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
[junit4] > at __randomizedtesting.SeedInfo.seed([23DB77BF5E4E778A]:0)
[junit4] 2> ??? 01, 2017 5:51:40 ? com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> ??? 01, 2017 5:52:00 ? com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.spatial3d.TestGeo3DPoint:
[junit4] 2> 1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[23DB77BF5E4E778A], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> at org.apache.lucene.spatial3d.geom.Plane.findIntersections(Plane.java:650)
[junit4] 2> at org.apache.lucene.spatial3d.geom.StandardXYZSolid.<init>(StandardXYZSolid.java:153)
[junit4] 2> at org.apache.lucene.spatial3d.geom.XYZSolidFactory.makeXYZSolid(XYZSolidFactory.java:64)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.verify(TestGeo3DPoint.java:847)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.doTestRandom(TestGeo3DPoint.java:548)
[junit4] 2> at org.apache.lucene.spatial3d.TestGeo3DPoint.testRandomBig(TestGeo3DPoint.java:480)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> ??? 01, 2017 5:52:00 ? com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=12, name=TEST-TestGeo3DPoint.testRandomBig-seed#[23DB77BF5E4E778A], state=RUNNABLE, group=TGRP-TestGeo3DPoint]
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/lucene/build/spatial3d/test/J1/temp/lucene.spatial3d.TestGeo3DPoint_23DB77BF5E4E778A-001
[junit4] 2> Aug 31, 2017 7:52:02 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: All leaked threads terminated.
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=Asserting)}, docValues:{id=DocValuesFormat(name=Direct), point=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1940, maxMBSortInHeap=7.655887576976247, sim=RandomSimilarity(queryNorm=true): {}, locale=ar-OM, timezone=Pacific/Guam
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=180741752,total=484442112
[junit4] 2> NOTE: All tests run in this JVM: [TestGeo3DPoint]
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestGeo3DPoint -Dtests.seed=23DB77BF5E4E778A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/test-data/enwiki.random.lines.txt -Dtests.locale=ar-OM -Dtests.timezone=Pacific/Guam -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 0.00s J1 | TestGeo3DPoint (suite) <<<
[junit4] > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
[junit4] > at __randomizedtesting.SeedInfo.seed([23DB77BF5E4E778A]:0)
[junit4] Completed [11/11 (1!)] on J1 in 7223.21s, 4 tests, 2 errors <<< FAILURES!
[...truncated 2778 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/init-core-data-001
[junit4] 2> 5301577 WARN (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=9 numCloses=9
[junit4] 2> 5301577 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 5301578 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 5301579 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-001
[junit4] 2> 5301579 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 5301589 INFO (Thread-73111) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 5301589 INFO (Thread-73111) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 5301646 ERROR (Thread-73111) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 5301689 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:33395
[junit4] 2> 5301799 INFO (jetty-launcher-1061-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 5301799 INFO (jetty-launcher-1061-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 5301874 INFO (jetty-launcher-1061-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@31b3a1ee{/solr,null,AVAILABLE}
[junit4] 2> 5301875 INFO (jetty-launcher-1061-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7999b4d2{HTTP/1.1,[http/1.1]}{127.0.0.1:35283}
[junit4] 2> 5301875 INFO (jetty-launcher-1061-thread-1) [ ] o.e.j.s.Server Started @5314197ms
[junit4] 2> 5301875 INFO (jetty-launcher-1061-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35283}
[junit4] 2> 5301875 ERROR (jetty-launcher-1061-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5301960 INFO (jetty-launcher-1061-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 5301960 INFO (jetty-launcher-1061-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5301960 INFO (jetty-launcher-1061-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5301960 INFO (jetty-launcher-1061-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-31T21:42:36.942Z
[junit4] 2> 5301955 INFO (jetty-launcher-1061-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@c467e30{/solr,null,AVAILABLE}
[junit4] 2> 5302024 INFO (jetty-launcher-1061-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@473fc035{HTTP/1.1,[http/1.1]}{127.0.0.1:44516}
[junit4] 2> 5302024 INFO (jetty-launcher-1061-thread-2) [ ] o.e.j.s.Server Started @5314347ms
[junit4] 2> 5302024 INFO (jetty-launcher-1061-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=44516}
[junit4] 2> 5302025 ERROR (jetty-launcher-1061-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5302025 INFO (jetty-launcher-1061-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 5302025 INFO (jetty-launcher-1061-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5302025 INFO (jetty-launcher-1061-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5302025 INFO (jetty-launcher-1061-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-31T21:42:37.007Z
[junit4] 2> 5302118 INFO (jetty-launcher-1061-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 5302119 INFO (jetty-launcher-1061-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 5302171 INFO (jetty-launcher-1061-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33395/solr
[junit4] 2> 5302218 INFO (jetty-launcher-1061-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33395/solr
[junit4] 2> 5302647 INFO (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5302711 INFO (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:44516_solr
[junit4] 2> 5302712 INFO (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr ] o.a.s.c.Overseer Overseer (id=98580283825324037-127.0.0.1:44516_solr-n_0000000000) starting
[junit4] 2> 5302761 INFO (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44516_solr
[junit4] 2> 5302938 INFO (OverseerStateUpdate-98580283825324037-127.0.0.1:44516_solr-n_0000000000) [n:127.0.0.1:44516_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5303040 INFO (jetty-launcher-1061-thread-1) [n:127.0.0.1:35283_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5303042 INFO (jetty-launcher-1061-thread-1) [n:127.0.0.1:35283_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5303057 INFO (jetty-launcher-1061-thread-1) [n:127.0.0.1:35283_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35283_solr
[junit4] 2> 5303129 INFO (zkCallback-1072-thread-2-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5303133 INFO (zkCallback-1073-thread-1-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5303259 INFO (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44516.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
[junit4] 2> 5303294 INFO (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44516.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
[junit4] 2> 5303294 INFO (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44516.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
[junit4] 2> 5303295 INFO (jetty-launcher-1061-thread-2) [n:127.0.0.1:44516_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-001/node2/.
[junit4] 2> 5303559 INFO (jetty-launcher-1061-thread-1) [n:127.0.0.1:35283_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35283.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
[junit4] 2> 5303590 INFO (jetty-launcher-1061-thread-1) [n:127.0.0.1:35283_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35283.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
[junit4] 2> 5303590 INFO (jetty-launcher-1061-thread-1) [n:127.0.0.1:35283_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35283.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
[junit4] 2> 5303591 INFO (jetty-launcher-1061-thread-1) [n:127.0.0.1:35283_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-001/node1/.
[junit4] 2> 5303894 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 5303895 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33395/solr ready
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 5304078 WARN (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 5304226 WARN (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 5304243 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 5304325 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_lucene2.us.west_apache_org_35259_hdfs____.tkjr36/webapp
[junit4] 2> 5305247 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@lucene2-us-west.apache.org:35259
[junit4] 2> 5305712 WARN (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 5305713 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 5305785 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_35554_datanode____bzkm3d/webapp
[junit4] 2> 5306820 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35554
[junit4] 2> 5307289 WARN (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 5307290 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 5307369 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_44438_datanode____f6yd1c/webapp
[junit4] 2> 5308449 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-002/hdfsBaseDir/data/data2/]] heartbeating to lucene2-us-west.apache.org/127.0.0.1:35262) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 5308518 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xf5da17843cc6d: from storage DS-0995900b-2a7f-4b81-959e-495da6b15206 node DatanodeRegistration(127.0.0.1:43963, datanodeUuid=3f58a021-348b-4320-93b0-377ff6d78662, infoPort=40459, infoSecurePort=0, ipcPort=40659, storageInfo=lv=-56;cid=testClusterID;nsid=1211097109;c=0), blocks: 0, hasStaleStorage: true, processing time: 33 msecs
[junit4] 2> 5308526 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xf5da17843cc6d: from storage DS-6330f322-6957-47ec-8012-2b0ef064fb59 node DatanodeRegistration(127.0.0.1:43963, datanodeUuid=3f58a021-348b-4320-93b0-377ff6d78662, infoPort=40459, infoSecurePort=0, ipcPort=40659, storageInfo=lv=-56;cid=testClusterID;nsid=1211097109;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 5308636 INFO (SUITE-HdfsRecoveryZkTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44438
[junit4] 2> 5309635 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-002/hdfsBaseDir/data/data4/]] heartbeating to lucene2-us-west.apache.org/127.0.0.1:35262) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 5309716 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xf5da1c118e971: from storage DS-32006cea-ed1b-4f2d-8150-815254924387 node DatanodeRegistration(127.0.0.1:44101, datanodeUuid=1205791d-58a4-428f-b8a0-2b1789d39e16, infoPort=34593, infoSecurePort=0, ipcPort=45338, storageInfo=lv=-56;cid=testClusterID;nsid=1211097109;c=0), blocks: 0, hasStaleStorage: true, processing time: 8 msecs
[junit4] 2> 5309716 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xf5da1c118e971: from storage DS-fc89b135-57f2-4ed5-90df-23bb75c813d5 node DatanodeRegistration(127.0.0.1:44101, datanodeUuid=1205791d-58a4-428f-b8a0-2b1789d39e16, infoPort=34593, infoSecurePort=0, ipcPort=45338, storageInfo=lv=-56;cid=testClusterID;nsid=1211097109;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 5310131 INFO (TEST-HdfsRecoveryZkTest.test-seed#[7FA8F1A6FFF1E22D]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 5310163 INFO (qtp1982336695-79706) [n:127.0.0.1:35283_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5310174 INFO (OverseerThreadFactory-2947-thread-1-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 5310314 INFO (OverseerStateUpdate-98580283825324037-127.0.0.1:44516_solr-n_0000000000) [n:127.0.0.1:44516_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"recoverytest",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"recoverytest_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:35283/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 5310377 INFO (OverseerStateUpdate-98580283825324037-127.0.0.1:44516_solr-n_0000000000) [n:127.0.0.1:44516_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"recoverytest",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"recoverytest_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:44516/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 5310751 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node1&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 5310752 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 5310817 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 5310818 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 5310939 INFO (zkCallback-1073-thread-1-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5310939 INFO (zkCallback-1072-thread-2-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5310939 INFO (zkCallback-1072-thread-1-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5310954 INFO (zkCallback-1073-thread-2-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5311830 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 5311891 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.s.IndexSchema [recoverytest_shard1_replica_n1] Schema name=minimal
[junit4] 2> 5311906 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 5311906 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica_n1' using configuration from collection recoverytest, trusted=true
[junit4] 2> 5311921 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35283.solr.core.recoverytest.shard1.replica_n1' (registry 'solr.core.recoverytest.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
[junit4] 2> 5311954 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://lucene2-us-west.apache.org:35262/data
[junit4] 2> 5311954 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 5311954 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5311954 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore [[recoverytest_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-001/node1/recoverytest_shard1_replica_n1], dataDir=[hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node1/data/]
[junit4] 2> 5311955 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node1/data/snapshot_metadata
[junit4] 2> 5312035 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 5312035 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 5312035 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 5312038 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 5312086 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.s.IndexSchema [recoverytest_shard1_replica_n2] Schema name=minimal
[junit4] 2> 5312140 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 5312140 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica_n2' using configuration from collection recoverytest, trusted=true
[junit4] 2> 5312148 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44516.solr.core.recoverytest.shard1.replica_n2' (registry 'solr.core.recoverytest.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14602bad
[junit4] 2> 5312149 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://lucene2-us-west.apache.org:35262/data
[junit4] 2> 5312149 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 5312149 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5312149 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore [[recoverytest_shard1_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7FA8F1A6FFF1E22D-001/tempDir-001/node2/recoverytest_shard1_replica_n2], dataDir=[hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node2/data/]
[junit4] 2> 5312150 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 5312151 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node1/data
[junit4] 2> 5312151 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node2/data/snapshot_metadata
[junit4] 2> 5312183 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 5312183 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 5312183 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 5312624 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 5312626 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node2/data
[junit4] 2> 5312750 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node1/data/index
[junit4] 2> 5312784 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 5312784 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 5312784 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 5312860 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node2/data/index
[junit4] 2> 5312877 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 5312877 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 5312877 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 5312928 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 5312946 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 5313331 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:44101 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-6330f322-6957-47ec-8012-2b0ef064fb59:NORMAL:127.0.0.1:43963|RBW], ReplicaUC[[DISK]DS-fc89b135-57f2-4ed5-90df-23bb75c813d5:NORMAL:127.0.0.1:44101|RBW]]} size 0
[junit4] 2> 5313378 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:43963 is added to blk_1073741825_1001 size 72
[junit4] 2> 5313378 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:43963 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-32006cea-ed1b-4f2d-8150-815254924387:NORMAL:127.0.0.1:44101|RBW], ReplicaUC[[DISK]DS-6330f322-6957-47ec-8012-2b0ef064fb59:NORMAL:127.0.0.1:43963|FINALIZED]]} size 0
[junit4] 2> 5313378 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:44101 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-32006cea-ed1b-4f2d-8150-815254924387:NORMAL:127.0.0.1:44101|RBW], ReplicaUC[[DISK]DS-6330f322-6957-47ec-8012-2b0ef064fb59:NORMAL:127.0.0.1:43963|FINALIZED]]} size 0
[junit4] 2> 5313798 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 5313798 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5313798 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 5313832 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5313832 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5313846 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 5313846 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5313847 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 5314022 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5314022 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5314074 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@72ae5b47[recoverytest_shard1_replica_n2] main]
[junit4] 2> 5314075 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 5314075 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 5314076 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 5314076 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577284554255761408
[junit4] 2> 5314091 INFO (searcherExecutor-2953-thread-1-processing-n:127.0.0.1:44516_solr x:recoverytest_shard1_replica_n2 s:shard1 c:recoverytest r:core_node2) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore [recoverytest_shard1_replica_n2] Registered new searcher Searcher@72ae5b47[recoverytest_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5314092 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@274525a2[recoverytest_shard1_replica_n1] main]
[junit4] 2> 5314093 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 5314093 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 5314093 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 5314094 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577284554274635776
[junit4] 2> 5314095 INFO (searcherExecutor-2952-thread-1-processing-n:127.0.0.1:35283_solr x:recoverytest_shard1_replica_n1 s:shard1 c:recoverytest r:core_node1) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore [recoverytest_shard1_replica_n1] Registered new searcher Searcher@274525a2[recoverytest_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5314156 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5314156 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 5314156 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/
[junit4] 2> 5314156 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica_n1 url=http://127.0.0.1:35283/solr START replicas=[http://127.0.0.1:44516/solr/recoverytest_shard1_replica_n2/] nUpdates=100
[junit4] 2> 5314156 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica_n1 url=http://127.0.0.1:35283/solr DONE. We have no versions. sync failed.
[junit4] 2> 5314157 INFO (zkCallback-1072-thread-2-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5314157 INFO (zkCallback-1072-thread-1-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5314158 INFO (zkCallback-1073-thread-2-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5314158 INFO (zkCallback-1073-thread-1-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5314192 INFO (qtp593564255-79713) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.c.S.Request [recoverytest_shard1_replica_n2] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=19
[junit4] 2> 5314204 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 5314204 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 5314204 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 5314206 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/ shard1
[junit4] 2> 5314342 INFO (zkCallback-1073-thread-1-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5314342 INFO (zkCallback-1073-thread-2-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5314342 INFO (zkCallback-1072-thread-2-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5314342 INFO (zkCallback-1072-thread-1-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5314370 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 5314375 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node1&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3623
[junit4] 2> 5314493 INFO (zkCallback-1073-thread-2-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5314493 INFO (zkCallback-1073-thread-1-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5314493 INFO (zkCallback-1072-thread-2-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5314493 INFO (zkCallback-1072-thread-1-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5315144 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=4327
[junit4] 2> 5315161 INFO (qtp1982336695-79706) [n:127.0.0.1:35283_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 5315305 INFO (zkCallback-1073-thread-1-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5315305 INFO (zkCallback-1073-thread-2-processing-n:127.0.0.1:35283_solr) [n:127.0.0.1:35283_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5315309 INFO (zkCallback-1072-thread-2-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5315309 INFO (zkCallback-1072-thread-1-processing-n:127.0.0.1:44516_solr) [n:127.0.0.1:44516_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5316167 INFO (qtp1982336695-79706) [n:127.0.0.1:35283_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=6003
[junit4] 2> 5316187 INFO (TEST-HdfsRecoveryZkTest.test-seed#[7FA8F1A6FFF1E22D]) [ ] o.a.s.c.RecoveryZkTest Indexing 3000 documents
[junit4] 2> 5316219 INFO (OverseerCollectionConfigSetProcessor-98580283825324037-127.0.0.1:44516_solr-n_0000000000) [n:127.0.0.1:44516_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5316455 DEBUG (qtp1982336695-79707) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.HdfsTransactionLog Opening new tlog hdfs tlog{file=hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node1/data/tlog/tlog.0000000000000000000 refcount=1}
[junit4] 2> 5316630 DEBUG (qtp593564255-79719) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.HdfsTransactionLog Opening new tlog hdfs tlog{file=hdfs://lucene2-us-west.apache.org:35262/data/recoverytest/core_node2/data/tlog/tlog.0000000000000000000 refcount=1}
[junit4] 2> 5316825 INFO (qtp593564255-79719) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-0 (1577284556768149504)]} 0 259
[junit4] 2> 5316825 INFO (qtp593564255-79717) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-0 (1577284556606668800)]} 0 325
[junit4] 2> 5316826 INFO (qtp1982336695-79707) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 (1577284556606668800)]} 0 507
[junit4] 2> 5316826 INFO (qtp1982336695-79708) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 (1577284556768149504)]} 0 354
[junit4] 2> 5316849 INFO (qtp593564255-79712) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-1 (1577284557139345408)]} 0 22
[junit4] 2> 5316849 INFO (qtp593564255-79713) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-1 (1577284557140393984)]} 0 21
[junit4] 2> 5316857 INFO (qtp1982336695-79710) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 (1577284557139345408)]} 0 31
[junit4] 2> 5316857 INFO (qtp1982336695-79709) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 (1577284557140393984)]} 0 30
[junit4] 2> 5316882 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-2 (1577284557172899840)]} 0 8
[junit4] 2> 5316882 INFO (qtp1982336695-79703) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 (1577284557172899840)]} 0 24
[junit4] 2> 5316882 INFO (qtp593564255-79718) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-2 (1577284557181288448)]} 0 8
[junit4] 2> 5316882 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 (1577284557181288448)]} 0 16
[junit4] 2> 5316916 INFO (qtp593564255-79719) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[1-0 (-1577284557214842880)]} 0 0
[junit4] 2> 5316927 INFO (qtp593564255-79717) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[2-0 (-1577284557214842881)]} 0 12
[junit4] 2> 5316927 INFO (qtp1982336695-79706) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-0 (-1577284557214842880)]} 0 29
[junit4] 2> 5316931 INFO (qtp1982336695-79707) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-0 (-1577284557214842881)]} 0 32
[junit4] 2> 5316932 INFO (qtp593564255-79712) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-3 (1577284557249445888)]} 0 0
[junit4] 2> 5316932 INFO (qtp1982336695-79708) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 (1577284557249445888)]} 0 1
[junit4] 2> 5316950 INFO (qtp593564255-79713) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[1-1 (-1577284557250494464)]} 0 1
[junit4] 2> 5316950 INFO (qtp1982336695-79707) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-1 (-1577284557250494464)]} 0 17
[junit4] 2> 5316963 INFO (qtp593564255-79718) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[1-4 (1577284557269368832)]} 0 12
[junit4] 2> 5316963 INFO (qtp593564255-79715) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-3 (1577284557266223104)]} 0 14
[junit4] 2> 5316964 INFO (qtp1982336695-79710) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 (1577284557266223104)]} 0 16
[junit4] 2> 5316978 INFO (qtp593564255-79719) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[2-1 (-1577284557296631808)]} 0 1
[junit4] 2> 5316978 INFO (qtp1982336695-79704) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-1 (-1577284557296631808)]} 0 2
[junit4] 2> 5316979 INFO (qtp1982336695-79703) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 (1577284557269368832)]} 0 29
[junit4] 2> 5317024 INFO (qtp593564255-79717) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{add=[2-4 (1577284557298728960)]} 0 45
[junit4] 2> 5317040 INFO (qtp1982336695-79709) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 (1577284557298728960)]} 0 62
[junit4] 2> 5317041 INFO (qtp593564255-79712) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[1-4 (-1577284557300826112)]} 0 0
[junit4] 2> 5317042 INFO (qtp593564255-79713) [n:127.0.0.1:44516_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35283/solr/recoverytest_shard1_replica_n1/&wt=javabin&version=2}{delete=[2-4 (-1577284557365837824)]} 0 0
[junit4] 2> 5317042 INFO (qtp1982336695-79708) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-4 (-1577284557365837824)]} 0 1
[junit4] 2> 5317044 INFO (qtp1982336695-79706) [n:127.0.0.1:35283_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update
[...truncated too long message...]
utoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:246)
[junit4] > at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:136)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 8057040 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 8057062 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 8057163 WARN (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to lucene2-us-west.apache.org/127.0.0.1:37596) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
[junit4] 2> 8057163 WARN (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to lucene2-us-west.apache.org/127.0.0.1:37596) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-601873968-127.0.0.1-1504218268494 (Datanode Uuid 383bbb35-e0e6-46b7-9d07-18524b62d8cc) service to lucene2-us-west.apache.org/127.0.0.1:37596
[junit4] 2> 8057164 WARN (SUITE-SharedFSAutoReplicaFailoverTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 8057265 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 8057366 WARN (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to lucene2-us-west.apache.org/127.0.0.1:37596) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
[junit4] 2> 8057366 WARN (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to lucene2-us-west.apache.org/127.0.0.1:37596) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-601873968-127.0.0.1-1504218268494 (Datanode Uuid 0ebdb639-c99b-4b07-b1e8-8cb6a76f5152) service to lucene2-us-west.apache.org/127.0.0.1:37596
[junit4] 2> 8057411 INFO (SUITE-SharedFSAutoReplicaFailoverTest-seed#[7FA8F1A6FFF1E22D]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@lucene2-us-west.apache.org:0
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.0/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.SharedFSAutoReplicaFailoverTest_7FA8F1A6FFF1E22D-001
[junit4] 2> Aug 31, 2017 10:28:32 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=PostingsFormat(name=LuceneVarGapDocFreqInterval), text=Lucene50(blocksize=128)}, docValues:{_version_=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1489, maxMBSortInHeap=7.226615572480204, sim=RandomSimilarity(queryNorm=true): {}, locale=ar-DZ, timezone=America/Edmonton
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=108331760,total=522715136
[junit4] 2> NOTE: All tests run in this JVM: [OverseerModifyCollectionTest, BitVectorTest, TestMaxScoreQueryParser, TestDFISimilarityFactory, DirectSolrSpellCheckerTest, TestRequestForwarding, IndexSchemaRuntimeFieldTest, HdfsThreadLeakTest, DistributedFacetExistsSmallTest, LoggingHandlerTest, ZkControllerTest, VersionInfoTest, AddBlockUpdateTest, DOMUtilTest, TlogReplayBufferedWhileIndexingTest, CacheHeaderTest, SoftAutoCommitTest, SliceStateTest, TestStressVersions, TestConfigSetsAPIExclusivity, ScriptEngineTest, DocValuesNotIndexedTest, TestLRUCache, TestSolrCloudWithDelegationTokens, TestAtomicUpdateErrorCases, ResourceLoaderTest, TestJsonFacetRefinement, FileBasedSpellCheckerTest, OverseerTaskQueueTest, TestReqParamsAPI, SpatialRPTFieldTypeTest, CheckHdfsIndexTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, DistributedMLTComponentTest, SortSpecParsingTest, ShardRoutingCustomTest, TestFiltering, BasicFunctionalityTest, BasicAuthIntegrationTest, SolrCoreMetricManagerTest, TestSha256AuthenticationProvider, TestManagedSynonymFilterFactory, TestConfigSets, DataDrivenBlockJoinTest, TestInPlaceUpdatesDistrib, TestCharFilters, TestDynamicFieldResource, SolrJmxReporterCloudTest, BlockJoinFacetDistribTest, TestValueSourceCache, TestSearchPerf, TestSchemaResource, TestSolrIndexConfig, StatelessScriptUpdateProcessorFactoryTest, DocValuesTest, TestApiFramework, SystemInfoHandlerTest, TestFieldCacheVsDocValues, TestMiniSolrCloudClusterSSL, SpatialFilterTest, TestTrieFacet, TestSmileRequest, JsonLoaderTest, TestDeleteCollectionOnDownNodes, FileUtilsTest, TestManagedResourceStorage, SchemaVersionSpecificBehaviorTest, MoveReplicaTest, DistributedIntervalFacetingTest, AutoScalingHandlerTest, FacetPivotSmallTest, DeleteStatusTest, TestHashQParserPlugin, TestSimpleTrackingShardHandler, HdfsBasicDistributedZkTest, SuggestComponentTest, ConvertedLegacyTest, ResponseHeaderTest, HighlighterTest, RuleEngineTest, TestPartialUpdateDeduplication, RemoteQueryErrorTest, CoreAdminHandlerTest, HdfsRecoveryZkTest, DistributedQueryComponentCustomSortTest, CoreSorterTest, CustomCollectionTest, TestFuzzyAnalyzedSuggestions, CSVRequestHandlerTest, LeaderInitiatedRecoveryOnCommitTest, TestClusterProperties, PrimUtilsTest, TestClassNameShortening, TestComplexPhraseLeadingWildcard, SOLR749Test, TestEmbeddedSolrServerSchemaAPI, TestPerFieldSimilarity, TestSystemCollAutoCreate, ShufflingReplicaListTransformerTest, MissingSegmentRecoveryTest, TestBM25SimilarityFactory, TestLegacyNumericUtils, TestLockTree, HdfsDirectoryFactoryTest, BlockDirectoryTest, TestSweetSpotSimilarityFactory, TestSlowCompositeReaderWrapper, CloudMLTQParserTest, TestFreeTextSuggestions, ManagedSchemaRoundRobinCloudTest, TestSQLHandlerNonCloud, UpdateRequestProcessorFactoryTest, TestFastLRUCache, TestCoreAdminApis, TestHttpShardHandlerFactory, TestExclusionRuleCollectionAccess, ConfigSetsAPITest, ChaosMonkeySafeLeaderTest, CollectionsAPIDistributedZkTest, OpenCloseCoreStressTest, ShardRoutingTest, BasicZkTest, ClusterStateUpdateTest, LeaderElectionTest, TestZkChroot, ZkCLITest, TestDistributedGrouping, TestHashPartitioner, TestStressReorder, TestReload, TestCoreContainer, SimpleFacetsTest, SpellCheckComponentTest, PeerSyncTest, TestLazyCores, DistributedQueryElevationComponentTest, AnalysisAfterCoreReloadTest, SuggesterFSTTest, SolrRequestParserTest, SuggesterTest, WordBreakSolrSpellCheckerTest, TestUpdate, DirectUpdateHandlerOptimizeTest, TestRemoteStreaming, DistanceFunctionTest, DebugComponentTest, LukeRequestHandlerTest, TestQueryTypes, TermVectorComponentTest, RequiredFieldsTest, TestSolrQueryParser, SolrPluginUtilsTest, TestCollationField, MBeansHandlerTest, TestAnalyzedSuggestions, AlternateDirectoryTest, TestQuerySenderNoQuery, SolrIndexConfigTest, BadComponentTest, TestStressRecovery, TestMergePolicyConfig, TestSolrDeletionPolicy2, SampleTest, TestDocSet, TestBinaryField, NumericFieldsTest, MinimalSchemaTest, TestElisionMultitermQuery, ExternalFileFieldSortTest, TestSolrCoreProperties, TestPostingsSolrHighlighter, DirectSolrConnectionTest, TestLuceneMatchVersion, TestPhraseSuggestions, TestXIncludeConfig, EchoParamsTest, TestDFRSimilarityFactory, TestIBSimilarityFactory, TestFastWriter, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, PluginInfoTest, ChaosMonkeyNothingIsSafeTest, CdcrVersionReplicationTest, CleanupOldIndexTest, CollectionReloadTest, CollectionsAPIAsyncDistributedZkTest, DistribCursorPagingTest, DistributedQueueTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, LeaderFailureAfterFreshStartTest, MultiThreadedOCPTest, SaslZkACLProviderTest, SharedFSAutoReplicaFailoverTest]
[junit4] Completed [523/729 (3!)] on J1 in 253.66s, 1 test, 1 failure <<< FAILURES!
[...truncated 44061 lines...]