You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2017/03/03 20:37:31 UTC
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1253 - Unstable
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1253/
3 tests failed.
FAILED: org.apache.solr.handler.extraction.ExtractingRequestHandlerTest.testExtraction
Error Message:
Exception during query
Stack Trace:
java.lang.RuntimeException: Exception during query
at __randomizedtesting.SeedInfo.seed([EE232EE8E5BD054B:57505A969995D5BE]:0)
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:919)
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:886)
at org.apache.solr.handler.extraction.ExtractingRequestHandlerTest.testExtraction(ExtractingRequestHandlerTest.java:128)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//doc[1]/str[.='simple3']
xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="2" start="0"><doc><arr name="t_meta"><str>stream_size</str><str>365</str><str>X-Parsed-By</str><str>org.apache.tika.parser.DefaultParser</str><str>X-Parsed-By</str><str>org.apache.tika.parser.html.HtmlParser</str><str>stream_content_type</str><str>application/xml</str><str>stream_name</str><str>simple.html</str><str>stream_source_info</str><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str><str>dc:title</str><str>Welcome to Solr</str><str>Content-Encoding</str><str>ISO-8859-1</str><str>Content-Type</str><str>text/html; charset=ISO-8859-1</str></arr><arr name="t_href"><str>rect</str><str>http://www.apache.org</str></arr><str name="id">simple2</str><arr name="stream_size"><str>365</str></arr><arr name="t_x_parsed_by"><str>org.apache.tika.parser.DefaultParser</str><str>org.apache.tika.parser.html.HtmlParser</str></arr><arr name="stream_content_type"><str>application/xml</str></arr><arr name="stream_name"><str>simple.html</str></arr><arr name="stream_source_info"><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str></arr><arr name="t_dc_title"><str>Welcome to Solr</str></arr><arr name="t_content_encoding"><str>ISO-8859-1</str></arr><arr name="title"><str>Welcome to Solr</str></arr><arr name="t_abcxyz"><str>text/html; charset=ISO-8859-1</str></arr><arr name="t_content"><str>
Welcome to Solr
Here is some text
distinct
words
Here is some text in a div
This has a link .
</str></arr><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2017-03-03T20:24:28.372Z</date></doc><doc><arr name="t_meta"><str>stream_size</str><str>365</str><str>X-Parsed-By</str><str>org.apache.tika.parser.DefaultParser</str><str>X-Parsed-By</str><str>org.apache.tika.parser.html.HtmlParser</str><str>stream_content_type</str><str>application/xml</str><str>stream_name</str><str>simple.html</str><str>stream_source_info</str><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str><str>dc:title</str><str>Welcome to Solr</str><str>Content-Encoding</str><str>ISO-8859-1</str><str>Content-Type</str><str>text/html; charset=ISO-8859-1</str></arr><arr name="t_href"><str>rect</str><str>http://www.apache.org</str></arr><str name="id">simple3</str><arr name="stream_size"><str>365</str></arr><arr name="t_x_parsed_by"><str>org.apache.tika.parser.DefaultParser</str><str>org.apache.tika.parser.html.HtmlParser</str></arr><arr name="stream_content_type"><str>application/xml</str></arr><arr name="stream_name"><str>simple.html</str></arr><arr name="stream_source_info"><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str></arr><arr name="t_dc_title"><str>Welcome to Solr</str></arr><arr name="t_content_encoding"><str>ISO-8859-1</str></arr><arr name="title"><str>Welcome to Solr</str></arr><arr name="t_content_type"><str>text/html; charset=ISO-8859-1</str></arr><arr name="t_content"><str>
Welcome to Solr
Here is some text
distinct
words
Here is some text in a div
This has a link .
</str></arr><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2017-03-03T20:24:28.487Z</date></doc></result>
</response>
request was:q=t_href:http&qt=standard&start=0&rows=20&version=2.2
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:912)
... 41 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.HdfsTransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130) at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:104) at sun.reflect.GeneratedConstructorAccessor207.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:759) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:821) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1071) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:936) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:829) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:937) at org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:572) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.HdfsTransactionLog
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:104)
at sun.reflect.GeneratedConstructorAccessor207.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:759)
at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:821)
at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1071)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:936)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:829)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:937)
at org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:572)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([D131E1FDF367CB3]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
at sun.reflect.GeneratedMethodAccessor108.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
FAILED: junit.framework.TestSuite.org.apache.solr.update.TestInPlaceUpdatesStandalone
Error Message:
Clean up static fields (in @AfterClass?), your test seems to hang on to approximately 22,002,080 bytes (threshold is 10,485,760). Field reference sizes (counted individually): - 22,001,752 bytes, private static org.apache.solr.client.solrj.SolrClient org.apache.solr.update.TestInPlaceUpdatesStandalone.client - 192 bytes, public static org.junit.rules.TestRule org.apache.solr.SolrTestCaseJ4.solrClassRules - 128 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.factoryProp - 64 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.coreName - 48 bytes, private static java.util.Map org.apache.solr.SolrTestCaseJ4.savedClassLogLevels
Stack Trace:
junit.framework.AssertionFailedError: Clean up static fields (in @AfterClass?), your test seems to hang on to approximately 22,002,080 bytes (threshold is 10,485,760). Field reference sizes (counted individually):
- 22,001,752 bytes, private static org.apache.solr.client.solrj.SolrClient org.apache.solr.update.TestInPlaceUpdatesStandalone.client
- 192 bytes, public static org.junit.rules.TestRule org.apache.solr.SolrTestCaseJ4.solrClassRules
- 128 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.factoryProp
- 64 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.coreName
- 48 bytes, private static java.util.Map org.apache.solr.SolrTestCaseJ4.savedClassLogLevels
at __randomizedtesting.SeedInfo.seed([D131E1FDF367CB3]:0)
at com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule$1.afterAlways(StaticFieldsInvariantRule.java:170)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 13145 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D131E1FDF367CB3-001/init-core-data-001
[junit4] 2> 3619508 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields
[junit4] 2> 3619509 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 3619510 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D131E1FDF367CB3-001/tempDir-001
[junit4] 2> 3619510 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3619510 INFO (Thread-40689) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3619510 INFO (Thread-40689) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3619610 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:56173
[junit4] 2> 3619615 INFO (jetty-launcher-3095-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3619615 INFO (jetty-launcher-3095-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3619616 INFO (jetty-launcher-3095-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5f74f634{/solr,null,AVAILABLE}
[junit4] 2> 3619617 INFO (jetty-launcher-3095-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@50c23115{HTTP/1.1,[http/1.1]}{127.0.0.1:38084}
[junit4] 2> 3619617 INFO (jetty-launcher-3095-thread-1) [ ] o.e.j.s.Server Started @3621959ms
[junit4] 2> 3619617 INFO (jetty-launcher-3095-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38084}
[junit4] 2> 3619617 ERROR (jetty-launcher-3095-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3619617 INFO (jetty-launcher-3095-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 3619617 INFO (jetty-launcher-3095-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3619617 INFO (jetty-launcher-3095-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3619617 INFO (jetty-launcher-3095-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-03T20:08:56.185Z
[junit4] 2> 3619618 INFO (jetty-launcher-3095-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@633c7b87{/solr,null,AVAILABLE}
[junit4] 2> 3619618 INFO (jetty-launcher-3095-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3390b519{HTTP/1.1,[http/1.1]}{127.0.0.1:56345}
[junit4] 2> 3619618 INFO (jetty-launcher-3095-thread-2) [ ] o.e.j.s.Server Started @3621960ms
[junit4] 2> 3619618 INFO (jetty-launcher-3095-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=56345}
[junit4] 2> 3619618 ERROR (jetty-launcher-3095-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3619619 INFO (jetty-launcher-3095-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 3619619 INFO (jetty-launcher-3095-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3619619 INFO (jetty-launcher-3095-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3619619 INFO (jetty-launcher-3095-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-03T20:08:56.187Z
[junit4] 2> 3619619 INFO (jetty-launcher-3095-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3619625 INFO (jetty-launcher-3095-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3619631 INFO (jetty-launcher-3095-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56173/solr
[junit4] 2> 3619632 INFO (jetty-launcher-3095-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56173/solr
[junit4] 2> 3619648 INFO (jetty-launcher-3095-thread-2) [n:127.0.0.1:56345_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3619648 INFO (jetty-launcher-3095-thread-1) [n:127.0.0.1:38084_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3619648 INFO (jetty-launcher-3095-thread-2) [n:127.0.0.1:56345_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56345_solr
[junit4] 2> 3619649 INFO (jetty-launcher-3095-thread-1) [n:127.0.0.1:38084_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38084_solr
[junit4] 2> 3619649 INFO (jetty-launcher-3095-thread-2) [n:127.0.0.1:56345_solr ] o.a.s.c.Overseer Overseer (id=97555037295673350-127.0.0.1:56345_solr-n_0000000000) starting
[junit4] 2> 3619650 INFO (zkCallback-3107-thread-1-processing-n:127.0.0.1:56345_solr) [n:127.0.0.1:56345_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3619650 INFO (zkCallback-3106-thread-1-processing-n:127.0.0.1:38084_solr) [n:127.0.0.1:38084_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3619664 INFO (jetty-launcher-3095-thread-2) [n:127.0.0.1:56345_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56345_solr
[junit4] 2> 3619665 INFO (zkCallback-3107-thread-1-processing-n:127.0.0.1:56345_solr) [n:127.0.0.1:56345_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3619665 INFO (zkCallback-3106-thread-1-processing-n:127.0.0.1:38084_solr) [n:127.0.0.1:38084_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3619687 INFO (jetty-launcher-3095-thread-1) [n:127.0.0.1:38084_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D131E1FDF367CB3-001/tempDir-001/node1/.
[junit4] 2> 3619743 INFO (jetty-launcher-3095-thread-2) [n:127.0.0.1:56345_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D131E1FDF367CB3-001/tempDir-001/node2/.
[junit4] 2> 3620577 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3620577 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56173/solr ready
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 3620607 WARN (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 3620611 WARN (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3620613 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3620623 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_44215_hdfs____.gzokjk/webapp
[junit4] 2> 3620944 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44215
[junit4] 2> 3621005 WARN (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3621006 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3621014 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_56802_datanode____wjv3og/webapp
[junit4] 2> 3621384 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:56802
[junit4] 2> 3621419 WARN (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3621420 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3621429 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_45051_datanode____.4pblym/webapp
[junit4] 2> 3621471 INFO (IPC Server handler 4 on 35882) [ ] BlockStateChange BLOCK* processReport: from storage DS-ad324f86-df3c-4c65-acc5-6dc0ec209410 node DatanodeRegistration(127.0.0.1:38037, datanodeUuid=31a98948-f154-46e5-9479-a25868a05ce1, infoPort=52146, infoSecurePort=0, ipcPort=32908, storageInfo=lv=-56;cid=testClusterID;nsid=1561559002;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3621471 INFO (IPC Server handler 4 on 35882) [ ] BlockStateChange BLOCK* processReport: from storage DS-bf1f4fe4-ef83-4717-b46b-40c5930b4a2a node DatanodeRegistration(127.0.0.1:38037, datanodeUuid=31a98948-f154-46e5-9479-a25868a05ce1, infoPort=52146, infoSecurePort=0, ipcPort=32908, storageInfo=lv=-56;cid=testClusterID;nsid=1561559002;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3621848 INFO (SUITE-HdfsRecoveryZkTest-seed#[D131E1FDF367CB3]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45051
[junit4] 2> 3621930 INFO (IPC Server handler 8 on 35882) [ ] BlockStateChange BLOCK* processReport: from storage DS-79f997cb-cf84-4288-8327-334d4505c388 node DatanodeRegistration(127.0.0.1:60714, datanodeUuid=7312cf2a-11eb-4dad-86d2-216f4285f9ce, infoPort=40459, infoSecurePort=0, ipcPort=33582, storageInfo=lv=-56;cid=testClusterID;nsid=1561559002;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3621930 INFO (IPC Server handler 8 on 35882) [ ] BlockStateChange BLOCK* processReport: from storage DS-2e2f9472-8949-4754-806b-72c93dde0352 node DatanodeRegistration(127.0.0.1:60714, datanodeUuid=7312cf2a-11eb-4dad-86d2-216f4285f9ce, infoPort=40459, infoSecurePort=0, ipcPort=33582, storageInfo=lv=-56;cid=testClusterID;nsid=1561559002;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3622031 INFO (TEST-HdfsRecoveryZkTest.test-seed#[D131E1FDF367CB3]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 3622032 INFO (qtp141765294-57538) [n:127.0.0.1:38084_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3622033 INFO (OverseerThreadFactory-8887-thread-1-processing-n:127.0.0.1:56345_solr) [n:127.0.0.1:56345_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 3622137 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 3622137 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 3622242 INFO (zkCallback-3106-thread-1-processing-n:127.0.0.1:38084_solr) [n:127.0.0.1:38084_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3622242 INFO (zkCallback-3107-thread-3-processing-n:127.0.0.1:56345_solr) [n:127.0.0.1:56345_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3623147 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 3623149 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 3623165 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema [recoverytest_shard1_replica2] Schema name=minimal
[junit4] 2> 3623166 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema [recoverytest_shard1_replica1] Schema name=minimal
[junit4] 2> 3623167 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3623168 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica2' using configuration from collection recoverytest
[junit4] 2> 3623168 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:35882/data
[junit4] 2> 3623168 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3623168 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3623168 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [[recoverytest_shard1_replica2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D131E1FDF367CB3-001/tempDir-001/node1/recoverytest_shard1_replica2], dataDir=[hdfs://localhost:35882/data/recoverytest/core_node2/data/]
[junit4] 2> 3623168 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6f70fb2c
[junit4] 2> 3623169 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica1' using configuration from collection recoverytest
[junit4] 2> 3623169 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35882/data/recoverytest/core_node2/data/snapshot_metadata
[junit4] 2> 3623169 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:35882/data
[junit4] 2> 3623169 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3623169 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [[recoverytest_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_D131E1FDF367CB3-001/tempDir-001/node2/recoverytest_shard1_replica1], dataDir=[hdfs://localhost:35882/data/recoverytest/core_node1/data/]
[junit4] 2> 3623169 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6f70fb2c
[junit4] 2> 3623169 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35882/data/recoverytest/core_node1/data/snapshot_metadata
[junit4] 2> 3623174 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3623174 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3623174 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3623174 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3623180 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3623182 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3623182 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35882/data/recoverytest/core_node2/data
[junit4] 2> 3623184 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35882/data/recoverytest/core_node1/data
[junit4] 2> 3623202 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35882/data/recoverytest/core_node2/data/index
[junit4] 2> 3623204 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:35882/data/recoverytest/core_node1/data/index
[junit4] 2> 3623208 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3623208 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3623209 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3623209 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3623211 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3623212 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3623249 INFO (IPC Server handler 8 on 35882) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:60714 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bf1f4fe4-ef83-4717-b46b-40c5930b4a2a:NORMAL:127.0.0.1:38037|RBW], ReplicaUC[[DISK]DS-79f997cb-cf84-4288-8327-334d4505c388:NORMAL:127.0.0.1:60714|RBW]]} size 0
[junit4] 2> 3623250 INFO (IPC Server handler 9 on 35882) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:38037 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-79f997cb-cf84-4288-8327-334d4505c388:NORMAL:127.0.0.1:60714|RBW], ReplicaUC[[DISK]DS-ad324f86-df3c-4c65-acc5-6dc0ec209410:NORMAL:127.0.0.1:38037|FINALIZED]]} size 0
[junit4] 2> 3623251 INFO (IPC Server handler 0 on 35882) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:38037 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-2e2f9472-8949-4754-806b-72c93dde0352:NORMAL:127.0.0.1:60714|RBW], ReplicaUC[[DISK]DS-bf1f4fe4-ef83-4717-b46b-40c5930b4a2a:NORMAL:127.0.0.1:38037|RBW]]} size 0
[junit4] 2> 3623253 INFO (IPC Server handler 6 on 35882) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:60714 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-2e2f9472-8949-4754-806b-72c93dde0352:NORMAL:127.0.0.1:60714|RBW], ReplicaUC[[DISK]DS-bf1f4fe4-ef83-4717-b46b-40c5930b4a2a:NORMAL:127.0.0.1:38037|RBW]]} size 0
[junit4] 2> 3623307 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 3623308 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3623308 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 3623321 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3623321 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3623334 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@7082e0ca[recoverytest_shard1_replica2] main]
[junit4] 2> 3623335 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 3623335 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 3623335 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3623335 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 3623335 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3623335 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 3623336 INFO (searcherExecutor-8892-thread-1-processing-n:127.0.0.1:38084_solr x:recoverytest_shard1_replica2 s:shard1 c:recoverytest) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [recoverytest_shard1_replica2] Registered new searcher Searcher@7082e0ca[recoverytest_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3623338 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1560880600743673856
[junit4] 2> 3623343 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 3623345 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3623345 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3623350 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@71ffecbc[recoverytest_shard1_replica1] main]
[junit4] 2> 3623350 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 3623351 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 3623351 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3623352 INFO (searcherExecutor-8893-thread-1-processing-n:127.0.0.1:56345_solr x:recoverytest_shard1_replica1 s:shard1 c:recoverytest) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [recoverytest_shard1_replica1] Registered new searcher Searcher@71ffecbc[recoverytest_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3623353 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1560880600759402496
[junit4] 2> 3623444 INFO (zkCallback-3107-thread-1-processing-n:127.0.0.1:56345_solr) [n:127.0.0.1:56345_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3623444 INFO (zkCallback-3106-thread-1-processing-n:127.0.0.1:38084_solr) [n:127.0.0.1:38084_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3623843 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3623844 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3623844 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/
[junit4] 2> 3623844 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:38084/solr START replicas=[http://127.0.0.1:56345/solr/recoverytest_shard1_replica1/] nUpdates=100
[junit4] 2> 3623844 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:38084/solr DONE. We have no versions. sync failed.
[junit4] 2> 3623845 INFO (qtp1310391835-57547) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.S.Request [recoverytest_shard1_replica1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3623845 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 3623845 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 3623845 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3623849 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/ shard1
[junit4] 2> 3623949 INFO (zkCallback-3106-thread-1-processing-n:127.0.0.1:38084_solr) [n:127.0.0.1:38084_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3623949 INFO (zkCallback-3107-thread-1-processing-n:127.0.0.1:56345_solr) [n:127.0.0.1:56345_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3623999 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3624001 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=1863
[junit4] 2> 3624102 INFO (zkCallback-3107-thread-1-processing-n:127.0.0.1:56345_solr) [n:127.0.0.1:56345_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3624102 INFO (zkCallback-3106-thread-1-processing-n:127.0.0.1:38084_solr) [n:127.0.0.1:38084_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3624360 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=2223
[junit4] 2> 3624362 INFO (qtp141765294-57538) [n:127.0.0.1:38084_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3624459 INFO (zkCallback-3107-thread-1-processing-n:127.0.0.1:56345_solr) [n:127.0.0.1:56345_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3624459 INFO (zkCallback-3106-thread-1-processing-n:127.0.0.1:38084_solr) [n:127.0.0.1:38084_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3625363 INFO (qtp141765294-57538) [n:127.0.0.1:38084_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=3330
[junit4] 2> 3625364 INFO (TEST-HdfsRecoveryZkTest.test-seed#[D131E1FDF367CB3]) [ ] o.a.s.c.RecoveryZkTest Indexing 3000 documents
[junit4] 2> 3625372 DEBUG (qtp141765294-57540) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.HdfsTransactionLog Opening new tlog hdfs tlog{file=hdfs://localhost:35882/data/recoverytest/core_node2/data/tlog/tlog.0000000000000000000 refcount=1}
[junit4] 2> 3625375 DEBUG (qtp1310391835-57549) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.HdfsTransactionLog Opening new tlog hdfs tlog{file=hdfs://localhost:35882/data/recoverytest/core_node1/data/tlog/tlog.0000000000000000000 refcount=1}
[junit4] 2> 3625379 INFO (qtp1310391835-57549) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-0 (1560880602873331712)]} 0 5
[junit4] 2> 3625379 INFO (qtp141765294-57540) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 (1560880602873331712)]} 0 10
[junit4] 2> 3625380 INFO (qtp1310391835-57550) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-1 (1560880602883817472)]} 0 0
[junit4] 2> 3625380 INFO (qtp141765294-57546) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 (1560880602883817472)]} 0 0
[junit4] 2> 3625381 INFO (qtp1310391835-57536) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-2 (1560880602885914624)]} 0 0
[junit4] 2> 3625381 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 (1560880602885914624)]} 0 0
[junit4] 2> 3625383 INFO (qtp1310391835-57539) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-3 (1560880602886963200)]} 0 0
[junit4] 2> 3625383 INFO (qtp141765294-57538) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 (1560880602886963200)]} 0 0
[junit4] 2> 3625384 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-4 (1560880602888011776)]} 0 0
[junit4] 2> 3625384 INFO (qtp141765294-57534) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 (1560880602888011776)]} 0 1
[junit4] 2> 3625384 INFO (qtp1310391835-57549) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-0 (1560880602875428864)]} 0 0
[junit4] 2> 3625385 INFO (qtp141765294-57544) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 (1560880602875428864)]} 0 14
[junit4] 2> 3625385 INFO (qtp1310391835-57550) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-5 (1560880602890108928)]} 0 0
[junit4] 2> 3625386 INFO (qtp141765294-57540) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 (1560880602890108928)]} 0 1
[junit4] 2> 3625386 INFO (qtp1310391835-57536) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-1 (1560880602890108929)]} 0 0
[junit4] 2> 3625387 INFO (qtp141765294-57542) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 (1560880602890108929)]} 0 1
[junit4] 2> 3625387 INFO (qtp1310391835-57550) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-6 (1560880602891157504)]} 0 0
[junit4] 2> 3625387 INFO (qtp141765294-57546) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 (1560880602891157504)]} 0 1
[junit4] 2> 3625388 INFO (qtp1310391835-57547) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-7 (1560880602893254656)]} 0 0
[junit4] 2> 3625389 INFO (qtp141765294-57538) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 (1560880602893254656)]} 0 0
[junit4] 2> 3625390 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-8 (1560880602894303232)]} 0 0
[junit4] 2> 3625390 INFO (qtp141765294-57544) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 (1560880602894303232)]} 0 0
[junit4] 2> 3625391 INFO (qtp1310391835-57549) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-9 (1560880602895351808)]} 0 0
[junit4] 2> 3625391 INFO (qtp141765294-57540) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 (1560880602895351808)]} 0 0
[junit4] 2> 3625392 INFO (qtp1310391835-57539) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-4 (-1560880602896400384)]} 0 0
[junit4] 2> 3625392 INFO (qtp141765294-57542) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-4 (-1560880602896400384)]} 0 1
[junit4] 2> 3625393 INFO (qtp1310391835-57536) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-10 (1560880602898497536)]} 0 0
[junit4] 2> 3625394 INFO (qtp141765294-57546) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 (1560880602898497536)]} 0 0
[junit4] 2> 3625395 INFO (qtp1310391835-57550) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-11 (1560880602899546112)]} 0 0
[junit4] 2> 3625395 INFO (qtp141765294-57538) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-11 (1560880602899546112)]} 0 0
[junit4] 2> 3625396 INFO (qtp1310391835-57547) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-12 (1560880602900594688)]} 0 0
[junit4] 2> 3625396 INFO (qtp141765294-57544) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-12 (1560880602900594688)]} 0 0
[junit4] 2> 3625397 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-13 (1560880602901643264)]} 0 0
[junit4] 2> 3625397 INFO (qtp141765294-57540) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-13 (1560880602901643264)]} 0 0
[junit4] 2> 3625398 INFO (qtp1310391835-57549) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-14 (1560880602902691840)]} 0 0
[junit4] 2> 3625398 INFO (qtp1310391835-57539) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-2 (1560880602892206080)]} 0 0
[junit4] 2> 3625398 INFO (qtp141765294-57542) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-14 (1560880602902691840)]} 0 0
[junit4] 2> 3625398 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 (1560880602892206080)]} 0 10
[junit4] 2> 3625400 INFO (qtp1310391835-57536) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-5 (-1560880602904788992)]} 0 0
[junit4] 2> 3625400 INFO (qtp1310391835-57550) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-3 (1560880602904788993)]} 0 0
[junit4] 2> 3625400 INFO (qtp141765294-57534) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-5 (-1560880602904788992)]} 0 1
[junit4] 2> 3625400 INFO (qtp141765294-57546) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 (1560880602904788993)]} 0 1
[junit4] 2> 3625403 INFO (qtp1310391835-57547) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-15 (1560880602906886144)]} 0 0
[junit4] 2> 3625403 INFO (qtp141765294-57544) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-15 (1560880602906886144)]} 0 2
[junit4] 2> 3625403 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-4 (1560880602906886145)]} 0 0
[junit4] 2> 3625404 INFO (qtp141765294-57538) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 (1560880602906886145)]} 0 3
[junit4] 2> 3625404 INFO (qtp1310391835-57549) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-6 (-1560880602910031872)]} 0 0
[junit4] 2> 3625405 INFO (qtp141765294-57542) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-6 (-1560880602910031872)]} 0 1
[junit4] 2> 3625405 INFO (qtp1310391835-57539) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-5 (1560880602910031873)]} 0 0
[junit4] 2> 3625405 INFO (qtp141765294-57540) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 (1560880602910031873)]} 0 0
[junit4] 2> 3625406 INFO (qtp1310391835-57536) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-16 (1560880602911080448)]} 0 0
[junit4] 2> 3625406 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-16 (1560880602911080448)]} 0 1
[junit4] 2> 3625406 INFO (qtp1310391835-57550) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-6 (1560880602911080449)]} 0 0
[junit4] 2> 3625407 INFO (qtp141765294-57534) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-6 (1560880602911080449)]} 0 1
[junit4] 2> 3625407 INFO (qtp1310391835-57547) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-17 (1560880602913177600)]} 0 0
[junit4] 2> 3625407 INFO (qtp141765294-57546) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-17 (1560880602913177600)]} 0 0
[junit4] 2> 3625408 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-7 (1560880602913177601)]} 0 0
[junit4] 2> 3625408 INFO (qtp141765294-57544) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-7 (1560880602913177601)]} 0 1
[junit4] 2> 3625409 INFO (qtp1310391835-57549) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-18 (1560880602914226176)]} 0 0
[junit4] 2> 3625409 INFO (qtp141765294-57538) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-18 (1560880602914226176)]} 0 1
[junit4] 2> 3625409 INFO (qtp1310391835-57539) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-8 (1560880602915274752)]} 0 0
[junit4] 2> 3625410 INFO (qtp141765294-57542) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-8 (1560880602915274752)]} 0 1
[junit4] 2> 3625410 INFO (qtp1310391835-57536) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-8 (-1560880602915274753)]} 0 0
[junit4] 2> 3625410 INFO (qtp141765294-57540) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-8 (-1560880602915274753)]} 0 1
[junit4] 2> 3625411 INFO (qtp1310391835-57539) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-9 (1560880602916323328)]} 0 0
[junit4] 2> 3625411 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-9 (1560880602916323328)]} 0 0
[junit4] 2> 3625412 INFO (qtp1310391835-57547) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-19 (1560880602917371904)]} 0 0
[junit4] 2> 3625412 INFO (qtp141765294-57534) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-19 (1560880602917371904)]} 0 1
[junit4] 2> 3625412 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-4 (-1560880602917371905)]} 0 0
[junit4] 2> 3625412 INFO (qtp141765294-57546) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-4 (-1560880602917371905)]} 0 1
[junit4] 2> 3625413 INFO (qtp1310391835-57547) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-20 (1560880602918420480)]} 0 0
[junit4] 2> 3625413 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-20 (1560880602918420480)]} 0 0
[junit4] 2> 3625413 INFO (qtp1310391835-57550) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-10 (1560880602919469056)]} 0 0
[junit4] 2> 3625414 INFO (qtp141765294-57538) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-10 (1560880602919469056)]} 0 0
[junit4] 2> 3625414 INFO (qtp1310391835-57536) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-21 (1560880602919469057)]} 0 0
[junit4] 2> 3625414 INFO (qtp141765294-57542) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-21 (1560880602919469057)]} 0 1
[junit4] 2> 3625415 INFO (qtp1310391835-57539) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-9 (-1560880602921566208)]} 0 0
[junit4] 2> 3625415 INFO (qtp141765294-57544) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-9 (-1560880602921566208)]} 0 0
[junit4] 2> 3625416 INFO (qtp1310391835-57549) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-22 (1560880602922614784)]} 0 0
[junit4] 2> 3625417 INFO (qtp141765294-57546) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-22 (1560880602922614784)]} 0 0
[junit4] 2> 3625417 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-23 (1560880602923663360)]} 0 0
[junit4] 2> 3625418 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-23 (1560880602923663360)]} 0 0
[junit4] 2> 3625419 INFO (qtp1310391835-57547) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-24 (1560880602924711936)]} 0 0
[junit4] 2> 3625419 INFO (qtp141765294-57538) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-24 (1560880602924711936)]} 0 0
[junit4] 2> 3625420 INFO (qtp1310391835-57550) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-25 (1560880602925760512)]} 0 0
[junit4] 2> 3625420 INFO (qtp141765294-57542) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-25 (1560880602925760512)]} 0 0
[junit4] 2> 3625421 INFO (qtp1310391835-57536) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-26 (1560880602926809088)]} 0 0
[junit4] 2> 3625421 INFO (qtp141765294-57544) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-26 (1560880602926809088)]} 0 0
[junit4] 2> 3625422 INFO (qtp1310391835-57539) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-27 (1560880602927857664)]} 0 0
[junit4] 2> 3625422 INFO (qtp141765294-57546) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-27 (1560880602927857664)]} 0 0
[junit4] 2> 3625423 INFO (qtp1310391835-57549) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-12 (-1560880602928906240)]} 0 0
[junit4] 2> 3625427 INFO (qtp141765294-57535) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-12 (-1560880602928906240)]} 0 5
[junit4] 2> 3625428 INFO (qtp1310391835-57543) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-11 (1560880602920517632)]} 0 0
[junit4] 2> 3625428 INFO (qtp141765294-57540) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-11 (1560880602920517632)]} 0 13
[junit4] 2> 3625428 INFO (qtp1310391835-57547) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-28 (1560880602935197696)]} 0 0
[junit4] 2> 3625429 INFO (qtp141765294-57538) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-28 (1560880602935197696)]} 0 1
[junit4] 2> 3625429 INFO (qtp1310391835-57550) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-12 (1560880602936246272)]} 0 0
[junit4] 2> 3625430 INFO (qtp141765294-57534) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-12 (1560880602936246272)]} 0 0
[junit4] 2> 3625430 INFO (qtp1310391835-57536) [n:127.0.0.1:56345_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:38084/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-29 (1560880602936246273)]} 0 0
[junit4] 2> 3625430 INFO (qtp141765294-57542) [n:127.0.0.1:38084_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2
[...truncated too long message...]
tionFailedError: Clean up static fields (in @AfterClass?), your test seems to hang on to approximately 22,002,080 bytes (threshold is 10,485,760). Field reference sizes (counted individually):
[junit4] > - 22,001,752 bytes, private static org.apache.solr.client.solrj.SolrClient org.apache.solr.update.TestInPlaceUpdatesStandalone.client
[junit4] > - 192 bytes, public static org.junit.rules.TestRule org.apache.solr.SolrTestCaseJ4.solrClassRules
[junit4] > - 128 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.factoryProp
[junit4] > - 64 bytes, private static java.lang.String org.apache.solr.SolrTestCaseJ4.coreName
[junit4] > - 48 bytes, private static java.util.Map org.apache.solr.SolrTestCaseJ4.savedClassLogLevels
[junit4] > at __randomizedtesting.SeedInfo.seed([D131E1FDF367CB3]:0)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] Completed [693/693 (2!)] on J2 in 171.37s, 18 tests, 1 failure <<< FAILURES!
[...truncated 5234 lines...]
[junit4] Suite: org.apache.solr.handler.extraction.ExtractingRequestHandlerTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-cell/test/J1/temp/solr.handler.extraction.ExtractingRequestHandlerTest_EE232EE8E5BD054B-001/init-core-data-001
[junit4] 2> log4j:WARN No appenders could be found for logger (org.apache.solr.SolrTestCaseJ4).
[junit4] 2> log4j:WARN Please initialize the log4j system properly.
[junit4] 2> log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ExtractingRequestHandlerTest -Dtests.method=testExtraction -Dtests.seed=EE232EE8E5BD054B -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=fr-BE -Dtests.timezone=SystemV/PST8 -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 2.33s J1 | ExtractingRequestHandlerTest.testExtraction <<<
[junit4] > Throwable #1: java.lang.RuntimeException: Exception during query
[junit4] > at __randomizedtesting.SeedInfo.seed([EE232EE8E5BD054B:57505A969995D5BE]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:919)
[junit4] > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:886)
[junit4] > at org.apache.solr.handler.extraction.ExtractingRequestHandlerTest.testExtraction(ExtractingRequestHandlerTest.java:128)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//doc[1]/str[.='simple3']
[junit4] > xml response was: <?xml version="1.0" encoding="UTF-8"?>
[junit4] > <response>
[junit4] > <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="2" start="0"><doc><arr name="t_meta"><str>stream_size</str><str>365</str><str>X-Parsed-By</str><str>org.apache.tika.parser.DefaultParser</str><str>X-Parsed-By</str><str>org.apache.tika.parser.html.HtmlParser</str><str>stream_content_type</str><str>application/xml</str><str>stream_name</str><str>simple.html</str><str>stream_source_info</str><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str><str>dc:title</str><str>Welcome to Solr</str><str>Content-Encoding</str><str>ISO-8859-1</str><str>Content-Type</str><str>text/html; charset=ISO-8859-1</str></arr><arr name="t_href"><str>rect</str><str>http://www.apache.org</str></arr><str name="id">simple2</str><arr name="stream_size"><str>365</str></arr><arr name="t_x_parsed_by"><str>org.apache.tika.parser.DefaultParser</str><str>org.apache.tika.parser.html.HtmlParser</str></arr><arr name="stream_content_type"><str>application/xml</str></arr><arr name="stream_name"><str>simple.html</str></arr><arr name="stream_source_info"><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str></arr><arr name="t_dc_title"><str>Welcome to Solr</str></arr><arr name="t_content_encoding"><str>ISO-8859-1</str></arr><arr name="title"><str>Welcome to Solr</str></arr><arr name="t_abcxyz"><str>text/html; charset=ISO-8859-1</str></arr><arr name="t_content"><str>
[junit4] >
[junit4] >
[junit4] >
[junit4] >
[junit4] >
[junit4] >
[junit4] >
[junit4] >
[junit4] >
[junit4] >
[junit4] > Welcome to Solr
[junit4] >
[junit4] >
[junit4] >
[junit4] > Here is some text
[junit4] >
[junit4] > distinct
[junit4] > words
[junit4] > Here is some text in a div
[junit4] > This has a link .
[junit4] > </str></arr><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2017-03-03T20:24:28.372Z</date></doc><doc><arr name="t_meta"><str>stream_size</str><str>365</str><str>X-Parsed-By</str><str>org.apache.tika.parser.DefaultParser</str><str>X-Parsed-By</str><str>org.apache.tika.parser.html.HtmlParser</str><str>stream_content_type</str><str>application/xml</str><str>stream_name</str><str>simple.html</str><str>stream_source_info</str><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str><str>dc:title</str><str>Welcome to Solr</str><str>Content-Encoding</str><str>ISO-8859-1</str><str>Content-Type</str><str>text/html; charset=ISO-8859-1</str></arr><arr name="t_href"><str>rect</str><str>http://www.apache.org</str></arr><str name="id">simple3</str><arr name="stream_size"><str>365</str></arr><arr name="t_x_parsed_by"><str>org.apache.tika.parser.DefaultParser</str><str>org.apache.tika.parser.html.HtmlParser</str></arr><arr name="stream_content_type"><str>application/xml</str></arr><arr name="stream_name"><str>simple.html</str></arr><arr name="stream_source_info"><str>file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/extraction/src/test-files/extraction/simple.html</str></arr><arr name="t_dc_title"><str>Welcome to Solr</str></arr><arr name="t_content_encoding"><str>ISO-8859-1</str></arr><arr name="title"><str>Welcome to Solr</str></arr><arr name="t_content_type"><str>text/html; charset=ISO-8859-1</str></arr><arr name="t_content"><str>
[junit4] >
[junit4] >
[junit4] >
[junit4] >
[junit4] >
[junit4] >
[junit4] >
[junit4] >
[junit4] >
[junit4] >
[junit4] > Welcome to Solr
[junit4] >
[junit4] >
[junit4] >
[junit4] > Here is some text
[junit4] >
[junit4] > distinct
[junit4] > words
[junit4] > Here is some text in a div
[junit4] > This has a link .
[junit4] > </str></arr><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2017-03-03T20:24:28.487Z</date></doc></result>
[junit4] > </response>
[junit4] > request was:q=t_href:http&qt=standard&start=0&rows=20&version=2.2
[junit4] > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:912)
[junit4] > ... 41 more
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-cell/test/J1/temp/solr.handler.extraction.ExtractingRequestHandlerTest_EE232EE8E5BD054B-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {extractionLiteralMV=BlockTreeOrds(blocksize=128), stream_content_type=BlockTreeOrds(blocksize=128), multiDefault=Lucene50(blocksize=128), subject=Lucene50(blocksize=128), intDefault=FST50, t_href=FST50, title=Lucene50(blocksize=128), extractedContent=BlockTreeOrds(blocksize=128), stream_name=FST50, t_X-Parsed-By=BlockTreeOrds(blocksize=128), bar_s=BlockTreeOrds(blocksize=128), Content-Encoding=BlockTreeOrds(blocksize=128), t_dc_title=BlockTreeOrds(blocksize=128), t_content_type=BlockTreeOrds(blocksize=128), extractedProducer=FST50, id=BlockTreeOrds(blocksize=128), text=PostingsFormat(name=LuceneVarGapFixedInterval), Content-Type=FST50, timestamp=FST50, stream_size=Lucene50(blocksize=128), extractedAuthor=BlockTreeOrds(blocksize=128), extractedKeywords=FST50, t_content_encoding=Lucene50(blocksize=128), stream_source_info=PostingsFormat(name=LuceneVarGapFixedInterval), t_content=FST50, resourceName=BlockTreeOrds(blocksize=128), foo_t=BlockTreeOrds(blocksize=128), foo_s=FST50, title_lettertok=BlockTreeOrds(blocksize=128), extractedCreator=Lucene50(blocksize=128), t_meta=Lucene50(blocksize=128), t_abcxyz=FST50, t_dc:title=Lucene50(blocksize=128), wdf_nocase=BlockTreeOrds(blocksize=128), extractedDate=FST50, div_t=FST50, title_stemmed=Lucene50(blocksize=128), anchor_t=FST50, t_x_parsed_by=PostingsFormat(name=LuceneVarGapFixedInterval), zot_s=BlockTreeOrds(blocksize=128), extractionLiteral=Lucene50(blocksize=128), defaultExtr=FST50}, docValues:{}, maxPointsInLeafNode=900, maxMBSortInHeap=5.592868203506101, sim=RandomSimilarity(queryNorm=false): {}, locale=fr-BE, timezone=SystemV/PST8
[junit4] 2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=4,threads=1,free=100355944,total=264241152
[junit4] 2> NOTE: All tests run in this JVM: [TestExtractionDateUtil, ExtractingRequestHandlerTest]
[junit4] Completed [4/4 (1!)] on J1 in 13.19s, 16 tests, 1 error <<< FAILURES!
[...truncated 57347 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1255 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1255/
2 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.HdfsTransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130) at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:104) at sun.reflect.GeneratedConstructorAccessor157.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:759) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:821) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1071) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:936) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:829) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:937) at org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:572) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.HdfsTransactionLog
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:104)
at sun.reflect.GeneratedConstructorAccessor157.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:759)
at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:821)
at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1071)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:936)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:829)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:937)
at org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:572)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([A5A0B1F6AC0C365C]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
at sun.reflect.GeneratedMethodAccessor80.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
FAILED: org.apache.solr.handler.admin.SegmentsInfoRequestHandlerTest.testSegmentInfosVersion
Error Message:
Exception during query
Stack Trace:
java.lang.RuntimeException: Exception during query
at __randomizedtesting.SeedInfo.seed([A5A0B1F6AC0C365C:5D7E241ED460E70F]:0)
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:919)
at org.apache.solr.handler.admin.SegmentsInfoRequestHandlerTest.testSegmentInfosVersion(SegmentsInfoRequestHandlerTest.java:70)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=2=count(//lst[@name='segments']/lst/str[@name='version'][.='7.0.0'])
xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">3</int></lst><lst name="segments"><lst name="_0"><str name="name">_0</str><int name="delCount">1</int><long name="sizeInBytes">1732</long><int name="size">4</int><date name="age">2017-03-05T17:35:43.518Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst><lst name="_1"><str name="name">_1</str><int name="delCount">0</int><long name="sizeInBytes">1581</long><int name="size">1</int><date name="age">2017-03-05T17:35:43.520Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst><lst name="_2"><str name="name">_2</str><int name="delCount">0</int><long name="sizeInBytes">1665</long><int name="size">4</int><date name="age">2017-03-05T17:35:43.527Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst><lst name="_3"><str name="name">_3</str><int name="delCount">0</int><long name="sizeInBytes">1581</long><int name="size">1</int><date name="age">2017-03-05T17:35:43.528Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst></lst>
</response>
request was:qt=/admin/segments&wt=xml
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:912)
... 40 more
Build Log:
[...truncated 13108 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A5A0B1F6AC0C365C-001/init-core-data-001
[junit4] 2> 3571238 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields
[junit4] 2> 3571239 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 3571239 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A5A0B1F6AC0C365C-001/tempDir-001
[junit4] 2> 3571239 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3571240 INFO (Thread-79513) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3571240 INFO (Thread-79513) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3571340 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:55857
[junit4] 2> 3571346 INFO (jetty-launcher-33238-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3571346 INFO (jetty-launcher-33238-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3571349 INFO (jetty-launcher-33238-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@54fd3937{/solr,null,AVAILABLE}
[junit4] 2> 3571350 INFO (jetty-launcher-33238-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@cd71c91{HTTP/1.1,[http/1.1]}{127.0.0.1:59336}
[junit4] 2> 3571350 INFO (jetty-launcher-33238-thread-2) [ ] o.e.j.s.Server Started @3573815ms
[junit4] 2> 3571350 INFO (jetty-launcher-33238-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59336}
[junit4] 2> 3571350 ERROR (jetty-launcher-33238-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3571351 INFO (jetty-launcher-33238-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 3571351 INFO (jetty-launcher-33238-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3571351 INFO (jetty-launcher-33238-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3571351 INFO (jetty-launcher-33238-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-05T17:33:15.615Z
[junit4] 2> 3571351 INFO (jetty-launcher-33238-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5721e3be{/solr,null,AVAILABLE}
[junit4] 2> 3571352 INFO (jetty-launcher-33238-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@78f49321{HTTP/1.1,[http/1.1]}{127.0.0.1:33936}
[junit4] 2> 3571352 INFO (jetty-launcher-33238-thread-1) [ ] o.e.j.s.Server Started @3573817ms
[junit4] 2> 3571352 INFO (jetty-launcher-33238-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33936}
[junit4] 2> 3571352 ERROR (jetty-launcher-33238-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3571352 INFO (jetty-launcher-33238-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 3571352 INFO (jetty-launcher-33238-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3571352 INFO (jetty-launcher-33238-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3571352 INFO (jetty-launcher-33238-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-05T17:33:15.616Z
[junit4] 2> 3571353 INFO (jetty-launcher-33238-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3571354 INFO (jetty-launcher-33238-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3571359 INFO (jetty-launcher-33238-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55857/solr
[junit4] 2> 3571359 INFO (jetty-launcher-33238-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55857/solr
[junit4] 2> 3571374 INFO (jetty-launcher-33238-thread-2) [n:127.0.0.1:59336_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3571374 INFO (jetty-launcher-33238-thread-1) [n:127.0.0.1:33936_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3571374 INFO (jetty-launcher-33238-thread-2) [n:127.0.0.1:59336_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59336_solr
[junit4] 2> 3571375 INFO (jetty-launcher-33238-thread-1) [n:127.0.0.1:33936_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33936_solr
[junit4] 2> 3571375 INFO (jetty-launcher-33238-thread-2) [n:127.0.0.1:59336_solr ] o.a.s.c.Overseer Overseer (id=97565749772550150-127.0.0.1:59336_solr-n_0000000000) starting
[junit4] 2> 3571387 INFO (zkCallback-33249-thread-1-processing-n:127.0.0.1:33936_solr) [n:127.0.0.1:33936_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3571387 INFO (zkCallback-33250-thread-1-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3571400 INFO (jetty-launcher-33238-thread-2) [n:127.0.0.1:59336_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59336_solr
[junit4] 2> 3571401 INFO (zkCallback-33249-thread-1-processing-n:127.0.0.1:33936_solr) [n:127.0.0.1:33936_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3571407 INFO (zkCallback-33250-thread-1-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3571424 INFO (jetty-launcher-33238-thread-1) [n:127.0.0.1:33936_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A5A0B1F6AC0C365C-001/tempDir-001/node1/.
[junit4] 2> 3571481 INFO (jetty-launcher-33238-thread-2) [n:127.0.0.1:59336_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A5A0B1F6AC0C365C-001/tempDir-001/node2/.
[junit4] 2> 3572239 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3572239 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55857/solr ready
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 3572267 WARN (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 3572277 WARN (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3572278 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3572288 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_51840_hdfs____46ljya/webapp
[junit4] 2> 3572621 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:51840
[junit4] 2> 3572673 WARN (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3572674 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3572683 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_48998_datanode____.f46osl/webapp
[junit4] 2> 3573077 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:48998
[junit4] 2> 3573127 WARN (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3573128 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3573137 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_53708_datanode____3u2q2w/webapp
[junit4] 2> 3573208 INFO (IPC Server handler 3 on 50422) [ ] BlockStateChange BLOCK* processReport: from storage DS-7e37e0d8-8022-4162-89a0-512a21232c5f node DatanodeRegistration(127.0.0.1:60707, datanodeUuid=5a741cd7-a89a-4458-a888-e844cd55c2d5, infoPort=43443, infoSecurePort=0, ipcPort=50407, storageInfo=lv=-56;cid=testClusterID;nsid=898209660;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3573208 INFO (IPC Server handler 3 on 50422) [ ] BlockStateChange BLOCK* processReport: from storage DS-a74a72c2-9aab-47bd-ab16-a688cef64514 node DatanodeRegistration(127.0.0.1:60707, datanodeUuid=5a741cd7-a89a-4458-a888-e844cd55c2d5, infoPort=43443, infoSecurePort=0, ipcPort=50407, storageInfo=lv=-56;cid=testClusterID;nsid=898209660;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3573540 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:53708
[junit4] 2> 3573612 INFO (IPC Server handler 8 on 50422) [ ] BlockStateChange BLOCK* processReport: from storage DS-3fac77ef-9841-4d7b-bd84-a6cb027ce3ad node DatanodeRegistration(127.0.0.1:41670, datanodeUuid=b958c4e3-95ea-4d33-a066-624181e0aef7, infoPort=32953, infoSecurePort=0, ipcPort=32961, storageInfo=lv=-56;cid=testClusterID;nsid=898209660;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3573612 INFO (IPC Server handler 8 on 50422) [ ] BlockStateChange BLOCK* processReport: from storage DS-e2131f17-47c5-481d-b327-961042eaabcc node DatanodeRegistration(127.0.0.1:41670, datanodeUuid=b958c4e3-95ea-4d33-a066-624181e0aef7, infoPort=32953, infoSecurePort=0, ipcPort=32961, storageInfo=lv=-56;cid=testClusterID;nsid=898209660;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3573746 INFO (TEST-HdfsRecoveryZkTest.test-seed#[A5A0B1F6AC0C365C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 3573747 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3573749 INFO (OverseerThreadFactory-15140-thread-1-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 3573855 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 3573855 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 3573961 INFO (zkCallback-33249-thread-1-processing-n:127.0.0.1:33936_solr) [n:127.0.0.1:33936_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3573961 INFO (zkCallback-33250-thread-3-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3574864 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 3574864 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 3574870 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema [recoverytest_shard1_replica2] Schema name=minimal
[junit4] 2> 3574870 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema [recoverytest_shard1_replica1] Schema name=minimal
[junit4] 2> 3574872 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3574872 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3574873 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica1' using configuration from collection recoverytest
[junit4] 2> 3574873 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica2' using configuration from collection recoverytest
[junit4] 2> 3574878 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:50422/data
[junit4] 2> 3574878 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3574878 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [[recoverytest_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A5A0B1F6AC0C365C-001/tempDir-001/node2/recoverytest_shard1_replica1], dataDir=[hdfs://localhost:50422/data/recoverytest/core_node2/data/]
[junit4] 2> 3574878 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:50422/data
[junit4] 2> 3574878 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3574878 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6fd8959d
[junit4] 2> 3574878 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [[recoverytest_shard1_replica2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A5A0B1F6AC0C365C-001/tempDir-001/node1/recoverytest_shard1_replica2], dataDir=[hdfs://localhost:50422/data/recoverytest/core_node1/data/]
[junit4] 2> 3574878 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6fd8959d
[junit4] 2> 3574878 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:50422/data/recoverytest/core_node2/data/snapshot_metadata
[junit4] 2> 3574879 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:50422/data/recoverytest/core_node1/data/snapshot_metadata
[junit4] 2> 3574885 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3574885 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3574885 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3574885 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3574887 WARN (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 3574887 WARN (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 3579893 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3579894 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3579895 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:50422/data/recoverytest/core_node2/data
[junit4] 2> 3579895 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:50422/data/recoverytest/core_node1/data
[junit4] 2> 3579909 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:50422/data/recoverytest/core_node2/data/index
[junit4] 2> 3579919 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:50422/data/recoverytest/core_node1/data/index
[junit4] 2> 3579920 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3579920 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3579929 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3579929 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3579929 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3579953 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3579978 INFO (IPC Server handler 9 on 50422) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41670 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-7e37e0d8-8022-4162-89a0-512a21232c5f:NORMAL:127.0.0.1:60707|RBW], ReplicaUC[[DISK]DS-3fac77ef-9841-4d7b-bd84-a6cb027ce3ad:NORMAL:127.0.0.1:41670|FINALIZED]]} size 0
[junit4] 2> 3579979 INFO (IPC Server handler 0 on 50422) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:60707 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-7e37e0d8-8022-4162-89a0-512a21232c5f:NORMAL:127.0.0.1:60707|RBW], ReplicaUC[[DISK]DS-3fac77ef-9841-4d7b-bd84-a6cb027ce3ad:NORMAL:127.0.0.1:41670|FINALIZED]]} size 0
[junit4] 2> 3579982 INFO (IPC Server handler 5 on 50422) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41670 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-e2131f17-47c5-481d-b327-961042eaabcc:NORMAL:127.0.0.1:41670|RBW], ReplicaUC[[DISK]DS-a74a72c2-9aab-47bd-ab16-a688cef64514:NORMAL:127.0.0.1:60707|RBW]]} size 0
[junit4] 2> 3579982 INFO (IPC Server handler 6 on 50422) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:60707 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-e2131f17-47c5-481d-b327-961042eaabcc:NORMAL:127.0.0.1:41670|RBW], ReplicaUC[[DISK]DS-a74a72c2-9aab-47bd-ab16-a688cef64514:NORMAL:127.0.0.1:60707|RBW]]} size 0
[junit4] 2> 3580027 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 3580027 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3580027 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 3580035 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3580035 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3580041 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 3580041 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3580041 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 3580049 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3580049 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3580052 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2d9fd695[recoverytest_shard1_replica1] main]
[junit4] 2> 3580053 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 3580054 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 3580054 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3580059 INFO (searcherExecutor-15145-thread-1-processing-n:127.0.0.1:59336_solr x:recoverytest_shard1_replica1 s:shard1 c:recoverytest) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [recoverytest_shard1_replica1] Registered new searcher Searcher@2d9fd695[recoverytest_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3580062 WARN (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:58117/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:58117 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3580062 WARN (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:58117/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:58117 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3580063 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1561052005612388352
[junit4] 2> 3580066 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@732ca366[recoverytest_shard1_replica2] main]
[junit4] 2> 3580067 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 3580067 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 3580067 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3580068 INFO (searcherExecutor-15146-thread-1-processing-n:127.0.0.1:33936_solr x:recoverytest_shard1_replica2 s:shard1 c:recoverytest) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [recoverytest_shard1_replica2] Registered new searcher Searcher@732ca366[recoverytest_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3580070 INFO (zkCallback-33250-thread-1-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3580070 WARN (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:58117/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:58117 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3580071 WARN (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:58117/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:58117 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3580071 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1561052005620776960
[junit4] 2> 3580071 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 3580073 INFO (zkCallback-33249-thread-1-processing-n:127.0.0.1:33936_solr) [n:127.0.0.1:33936_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3580591 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3580592 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3580592 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/
[junit4] 2> 3580592 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica1 url=http://127.0.0.1:59336/solr START replicas=[http://127.0.0.1:33936/solr/recoverytest_shard1_replica2/] nUpdates=100
[junit4] 2> 3580592 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica1 url=http://127.0.0.1:59336/solr DONE. We have no versions. sync failed.
[junit4] 2> 3580605 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.S.Request [recoverytest_shard1_replica2] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 3580606 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 3580606 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 3580606 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3580609 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/ shard1
[junit4] 2> 3580713 INFO (zkCallback-33249-thread-1-processing-n:127.0.0.1:33936_solr) [n:127.0.0.1:33936_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3580713 INFO (zkCallback-33250-thread-3-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3580759 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3580762 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=6906
[junit4] 2> 3580865 INFO (zkCallback-33249-thread-1-processing-n:127.0.0.1:33936_solr) [n:127.0.0.1:33936_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3580865 INFO (zkCallback-33250-thread-2-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3581076 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=7221
[junit4] 2> 3581079 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3581179 INFO (zkCallback-33249-thread-1-processing-n:127.0.0.1:33936_solr) [n:127.0.0.1:33936_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3581179 INFO (zkCallback-33250-thread-3-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3581753 INFO (OverseerCollectionConfigSetProcessor-97565749772550150-127.0.0.1:59336_solr-n_0000000000) [n:127.0.0.1:59336_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3582080 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=8332
[junit4] 2> 3582081 INFO (TEST-HdfsRecoveryZkTest.test-seed#[A5A0B1F6AC0C365C]) [ ] o.a.s.c.RecoveryZkTest Indexing 12000 documents
[junit4] 2> 3582090 DEBUG (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.HdfsTransactionLog Opening new tlog hdfs tlog{file=hdfs://localhost:50422/data/recoverytest/core_node2/data/tlog/tlog.0000000000000000000 refcount=1}
[junit4] 2> 3582096 DEBUG (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.HdfsTransactionLog Opening new tlog hdfs tlog{file=hdfs://localhost:50422/data/recoverytest/core_node1/data/tlog/tlog.0000000000000000000 refcount=1}
[junit4] 2> 3582100 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-0 (1561052007732609024)]} 0 7
[junit4] 2> 3582100 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 (1561052007732609024)]} 0 15
[junit4] 2> 3582101 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-0 (1561052007731560448)]} 0 9
[junit4] 2> 3582101 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 (1561052007731560448)]} 0 17
[junit4] 2> 3582103 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-0 (-1561052007749386240)]} 0 1
[junit4] 2> 3582104 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-0 (-1561052007749386240)]} 0 2
[junit4] 2> 3582104 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-0 (-1561052007750434816)]} 0 0
[junit4] 2> 3582104 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-0 (-1561052007750434816)]} 0 2
[junit4] 2> 3582105 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-1 (1561052007752531968)]} 0 0
[junit4] 2> 3582106 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 (1561052007752531968)]} 0 1
[junit4] 2> 3582106 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-1 (1561052007753580544)]} 0 0
[junit4] 2> 3582107 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 (1561052007753580544)]} 0 2
[junit4] 2> 3582107 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-2 (1561052007754629120)]} 0 0
[junit4] 2> 3582107 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 (1561052007754629120)]} 0 1
[junit4] 2> 3582109 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-2 (1561052007755677696)]} 0 0
[junit4] 2> 3582109 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 (1561052007755677696)]} 0 1
[junit4] 2> 3582109 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-1 (-1561052007756726272)]} 0 0
[junit4] 2> 3582109 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-1 (-1561052007756726272)]} 0 1
[junit4] 2> 3582111 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-1 (-1561052007758823424)]} 0 1
[junit4] 2> 3582111 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-3 (1561052007758823425)]} 0 0
[junit4] 2> 3582111 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-1 (-1561052007758823424)]} 0 1
[junit4] 2> 3582111 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 (1561052007758823425)]} 0 1
[junit4] 2> 3582113 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-4 (1561052007760920577)]} 0 0
[junit4] 2> 3582113 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 (1561052007760920577)]} 0 1
[junit4] 2> 3582113 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-3 (1561052007760920576)]} 0 0
[junit4] 2> 3582114 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 (1561052007760920576)]} 0 1
[junit4] 2> 3582115 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-5 (1561052007763017728)]} 0 0
[junit4] 2> 3582115 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 (1561052007763017728)]} 0 1
[junit4] 2> 3582119 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-4 (1561052007763017729)]} 0 3
[junit4] 2> 3582119 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 (1561052007763017729)]} 0 4
[junit4] 2> 3582119 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-3 (-1561052007765114880)]} 0 3
[junit4] 2> 3582119 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-3 (-1561052007765114880)]} 0 3
[junit4] 2> 3582120 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-6 (1561052007769309184)]} 0 0
[junit4] 2> 3582120 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 (1561052007769309184)]} 0 0
[junit4] 2> 3582120 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-5 (1561052007768260608)]} 0 0
[junit4] 2> 3582121 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 (1561052007768260608)]} 0 1
[junit4] 2> 3582122 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-4 (-1561052007770357760)]} 0 0
[junit4] 2> 3582122 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-4 (-1561052007770357760)]} 0 0
[junit4] 2> 3582122 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-3 (-1561052007770357761)]} 0 1
[junit4] 2> 3582123 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-3 (-1561052007770357761)]} 0 1
[junit4] 2> 3582123 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-7 (1561052007771406336)]} 0 0
[junit4] 2> 3582124 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 (1561052007771406336)]} 0 1
[junit4] 2> 3582124 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-6 (1561052007772454912)]} 0 0
[junit4] 2> 3582124 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-6 (1561052007772454912)]} 0 1
[junit4] 2> 3582125 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-8 (1561052007773503488)]} 0 0
[junit4] 2> 3582125 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 (1561052007773503488)]} 0 1
[junit4] 2> 3582126 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-4 (-1561052007774552064)]} 0 0
[junit4] 2> 3582126 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-4 (-1561052007774552064)]} 0 1
[junit4] 2> 3582129 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-6 (-1561052007775600640)]} 0 1
[junit4] 2> 3582129 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-6 (-1561052007775600640)]} 0 3
[junit4] 2> 3582129 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-7 (1561052007776649216)]} 0 1
[junit4] 2> 3582129 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-7 (1561052007776649216)]} 0 2
[junit4] 2> 3582131 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-9 (1561052007778746368)]} 0 0
[junit4] 2> 3582131 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 (1561052007778746368)]} 0 1
[junit4] 2> 3582131 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-8 (1561052007779794944)]} 0 0
[junit4] 2> 3582132 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-8 (1561052007779794944)]} 0 2
[junit4] 2> 3582132 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-8 (-1561052007780843520)]} 0 0
[junit4] 2> 3582133 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-8 (-1561052007780843520)]} 0 1
[junit4] 2> 3582133 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-6 (-1561052007781892096)]} 0 0
[junit4] 2> 3582133 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-6 (-1561052007781892096)]} 0 1
[junit4] 2> 3582135 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-10 (1561052007782940672)]} 0 2
[junit4] 2> 3582136 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 (1561052007782940672)]} 0 2
[junit4] 2> 3582138 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-9 (-1561052007786086400)]} 0 0
[junit4] 2> 3582138 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-9 (-1561052007786086400)]} 0 1
[junit4] 2> 3582138 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-9 (1561052007786086401)]} 0 1
[junit4] 2> 3582138 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-9 (1561052007786086401)]} 0 1
[junit4] 2> 3582139 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-11 (1561052007788183552)]} 0 0
[junit4] 2> 3582139 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-8 (-1561052007789232128)]} 0 0
[junit4] 2> 3582139 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-11 (1561052007788183552)]} 0 1
[junit4] 2> 3582139 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-8 (-1561052007789232128)]} 0 0
[junit4] 2> 3582141 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-10 (-1561052007790280704)]} 0 0
[junit4] 2> 3582141 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-10 (-1561052007790280704)]} 0 1
[junit4] 2> 3582141 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-10 (1561052007790280705)]} 0 0
[junit4] 2> 3582142 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-10 (1561052007790280705)]} 0 1
[junit4] 2> 3582143 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-12 (1561052007791329280)]} 0 0
[junit4] 2> 3582143 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-12 (1561052007791329280)]} 0 1
[junit4] 2> 3582143 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-9 (-1561052007792377856)]} 0 0
[junit4] 2> 3582143 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-9 (-1561052007792377856)]} 0 1
[junit4] 2> 3582144 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-12 (-1561052007793426432)]} 0 0
[junit4] 2> 3582145 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-12 (-1561052007793426432)]} 0 1
[junit4] 2> 3582145 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-11 (1561052007794475008)]} 0 0
[junit4] 2> 3582145 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-11 (1561052007794475008)]} 0 1
[junit4] 2> 3582146 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-13 (1561052007795523585)]} 0 0
[junit4] 2> 3582146 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-13 (1561052007795523585)]} 0 1
[junit4] 2> 3582147 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-10 (-1561052007795523584)]} 0 0
[junit4] 2> 3582147 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-10 (-1561052007795523584)]} 0 1
[junit4] 2> 3582148 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-13 (-1561052007797620736)]} 0 0
[junit4] 2> 3582148 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-13 (-1561052007797620736)]} 0 1
[junit4] 2> 3582148 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-12 (1561052007797620737)]} 0 0
[junit4] 2> 3582149 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-12 (1561052007797620737)]} 0 1
[junit4] 2> 3582150 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-14 (1561052007799717888)]} 0 0
[junit4] 2> 3582150 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-14 (1561052007799717888)]} 0 1
[junit4] 2> 3582150 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-12 (-1561052007799717889)]} 0 0
[junit4] 2> 3582150 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-12 (-1561052007799717889)]} 0 1
[junit4] 2> 3582152 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-15 (1561052007801815040)]} 0 0
[junit4] 2> 3582152 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-15 (1561052007801815040)]} 0 1
[junit4] 2> 3582152 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-13 (1561052007801815041)]} 0 0
[junit4] 2> 3582153 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-13 (1561052007801815041)]} 0 1
[junit4] 2> 3582154 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-14 (-1561052007803912192)]} 0 0
[junit4] 2> 3582154 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-14 (-1561052007803912192)]} 0 1
[junit4] 2> 3582154 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-13 (-1561052007803912193)]} 0 0
[junit4] 2> 3582154 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-13 (-1561052007803912193)]} 0 1
[junit4] 2> 3582155 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-16 (1561052007804960768)]} 0 0
[junit4] 2> 3582156 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-16 (1561052007804960768)]} 0 1
[junit4] 2> 3582156 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-14 (1561052007806009344)]} 0 0
[junit4] 2> 3582157 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-14 (1561052007806009344)]} 0 1
[junit4] 2> 3582157 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.Log
[...truncated too long message...]
h.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 3718928 INFO (TEST-SegmentsInfoRequestHandlerTest.testSegmentInfosVersion-seed#[A5A0B1F6AC0C365C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSegmentInfosVersion
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=SegmentsInfoRequestHandlerTest -Dtests.method=testSegmentInfosVersion -Dtests.seed=A5A0B1F6AC0C365C -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=zh -Dtests.timezone=Brazil/Acre -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.01s J1 | SegmentsInfoRequestHandlerTest.testSegmentInfosVersion <<<
[junit4] > Throwable #1: java.lang.RuntimeException: Exception during query
[junit4] > at __randomizedtesting.SeedInfo.seed([A5A0B1F6AC0C365C:5D7E241ED460E70F]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:919)
[junit4] > at org.apache.solr.handler.admin.SegmentsInfoRequestHandlerTest.testSegmentInfosVersion(SegmentsInfoRequestHandlerTest.java:70)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=2=count(//lst[@name='segments']/lst/str[@name='version'][.='7.0.0'])
[junit4] > xml response was: <?xml version="1.0" encoding="UTF-8"?>
[junit4] > <response>
[junit4] > <lst name="responseHeader"><int name="status">0</int><int name="QTime">3</int></lst><lst name="segments"><lst name="_0"><str name="name">_0</str><int name="delCount">1</int><long name="sizeInBytes">1732</long><int name="size">4</int><date name="age">2017-03-05T17:35:43.518Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst><lst name="_1"><str name="name">_1</str><int name="delCount">0</int><long name="sizeInBytes">1581</long><int name="size">1</int><date name="age">2017-03-05T17:35:43.520Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst><lst name="_2"><str name="name">_2</str><int name="delCount">0</int><long name="sizeInBytes">1665</long><int name="size">4</int><date name="age">2017-03-05T17:35:43.527Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst><lst name="_3"><str name="name">_3</str><int name="delCount">0</int><long name="sizeInBytes">1581</long><int name="size">1</int><date name="age">2017-03-05T17:35:43.528Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst></lst>
[junit4] > </response>
[junit4] > request was:qt=/admin/segments&wt=xml
[junit4] > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:912)
[junit4] > ... 40 more
[junit4] 2> 3718934 INFO (TEST-SegmentsInfoRequestHandlerTest.testSegmentInfos-seed#[A5A0B1F6AC0C365C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSegmentInfos
[junit4] 2> 3718934 INFO (TEST-SegmentsInfoRequestHandlerTest.testSegmentInfos-seed#[A5A0B1F6AC0C365C]) [ ] o.a.s.c.S.Request [collection1] webapp=null path=null params={qt=/admin/segments&wt=xml} status=0 QTime=0
[junit4] 2> 3718935 INFO (TEST-SegmentsInfoRequestHandlerTest.testSegmentInfos-seed#[A5A0B1F6AC0C365C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSegmentInfos
[junit4] 2> 3718935 INFO (SUITE-SegmentsInfoRequestHandlerTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> 3718935 INFO (SUITE-SegmentsInfoRequestHandlerTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1549558850
[junit4] 2> 3718935 INFO (coreCloseExecutor-7622-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@493e70cf
[junit4] 2> 3718957 INFO (coreCloseExecutor-7622-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 3718959 INFO (SUITE-SegmentsInfoRequestHandlerTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.handler.admin.SegmentsInfoRequestHandlerTest_A5A0B1F6AC0C365C-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {name=FSTOrd50, id=FSTOrd50}, docValues:{}, maxPointsInLeafNode=710, maxMBSortInHeap=6.510209896215786, sim=RandomSimilarity(queryNorm=true): {}, locale=zh, timezone=Brazil/Acre
[junit4] 2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=4,threads=1,free=208588576,total=529530880
[junit4] 2> NOTE: All tests run in this JVM: [DirectSolrSpellCheckerTest, TestClassNameShortening, CleanupOldIndexTest, TestSchemaNameResource, HdfsCollectionsAPIDistributedZkTest, CollectionStateFormat2Test, TestDeleteCollectionOnDownNodes, TriLevelCompositeIdRoutingTest, CdcrRequestHandlerTest, TestExclusionRuleCollectionAccess, UpdateRequestProcessorFactoryTest, MoreLikeThisHandlerTest, UUIDUpdateProcessorFallbackTest, TestManagedSynonymFilterFactory, TestDocBasedVersionConstraints, SampleTest, AddBlockUpdateTest, HdfsLockFactoryTest, TestNRTOpen, TestRandomRequestDistribution, TestObjectReleaseTracker, TestSolrCloudWithDelegationTokens, CSVRequestHandlerTest, DistributedFacetPivotSmallTest, CheckHdfsIndexTest, InfoHandlerTest, TestStressUserVersions, TestRandomCollapseQParserPlugin, EnumFieldTest, BlockJoinFacetSimpleTest, TestUpdate, ShardRoutingTest, OverseerStatusTest, TestPartialUpdateDeduplication, TestScoreJoinQPScore, TestSurroundQueryParser, TestRecoveryHdfs, TestXIncludeConfig, SpellCheckCollatorTest, HdfsWriteToMultipleCollectionsTest, SpatialFilterTest, ChaosMonkeySafeLeaderTest, SchemaVersionSpecificBehaviorTest, TestSimpleQParserPlugin, TestAnalyzeInfixSuggestions, ZkCLITest, DistributedDebugComponentTest, IndexSchemaTest, BJQParserTest, ZkSolrClientTest, TermsComponentTest, TestFuzzyAnalyzedSuggestions, DistributedMLTComponentTest, TestQueryUtils, TestSystemIdResolver, TestSQLHandler, VersionInfoTest, CoreAdminCreateDiscoverTest, TestManagedSchema, TestReplicationHandlerBackup, TestManagedResourceStorage, HdfsBasicDistributedZkTest, DocValuesTest, TestBinaryResponseWriter, SolrCmdDistributorTest, TestSolrQueryResponse, TestFreeTextSuggestions, InfixSuggestersTest, SpellingQueryConverterTest, SolrInfoMBeanTest, TestUnifiedSolrHighlighter, DistributedFacetPivotLargeTest, TestConfigSetImmutable, TestFieldResource, SolrIndexSplitterTest, ConnectionManagerTest, TestSweetSpotSimilarityFactory, TestPointFields, TestClusterProperties, IndexSchemaRuntimeFieldTest, TestRequestForwarding, TestEmbeddedSolrServerConstructors, DistanceFunctionTest, DistributedQueryComponentOptimizationTest, SimplePostToolTest, TestSolrConfigHandler, AlternateDirectoryTest, TestMiniSolrCloudClusterSSL, TestDefaultStatsCache, TestCursorMarkWithoutUniqueKey, SharedFSAutoReplicaFailoverTest, ZkStateReaderTest, QueryParsingTest, TestNumericTerms32, DistributedQueueTest, TestCloudPivotFacet, TestHashPartitioner, TestWordDelimiterFilterFactory, LeaderInitiatedRecoveryOnShardRestartTest, DOMUtilTest, TestBlobHandler, TestStressCloudBlindAtomicUpdates, SolrJmxReporterTest, BasicFunctionalityTest, DistributedTermsComponentTest, TestPostingsSolrHighlighter, ReplicationFactorTest, TestDocumentBuilder, DistributedIntervalFacetingTest, SolrCLIZkUtilsTest, TestLegacyNumericRangeQueryBuilder, HighlighterTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerTaskQueueTest, RemoteQueryErrorTest, RestartWhileUpdatingTest, SharedFSAutoReplicaFailoverUtilsTest, SolrCloudExampleTest, TestAuthenticationFramework, TestCloudDeleteByQuery, TestCloudInspectUtil, TestConfigSetsAPIExclusivity, TestCryptoKeys, TestDistribDocBasedVersion, TestLeaderElectionZkExpiry, TestLockTree, TestMiniSolrCloudCluster, TestRandomFlRTGCloud, TestSolrCloudWithSecureImpersonation, HdfsNNFailoverTest, HdfsThreadLeakTest, StressHdfsTest, RuleEngineTest, BlobRepositoryCloudTest, TestDynamicLoading, RequestLoggingTest, TestConfigReload, SegmentsInfoRequestHandlerTest]
[junit4] Completed [561/693 (2!)] on J1 in 0.38s, 3 tests, 1 error <<< FAILURES!
[...truncated 63018 lines...]
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1254 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1254/
2 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 1) Thread[id=3211, name=qtp510760146-3211, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Throwable.fillInStackTrace(Native Method) at java.lang.Throwable.fillInStackTrace(Throwable.java:783) at java.lang.Throwable.<init>(Throwable.java:250) at java.lang.Exception.<init>(Exception.java:54) at java.lang.InterruptedException.<init>(InterruptedException.java:57) at java.lang.Object.wait(Native Method) at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.waitForEmptyQueue(ConcurrentUpdateSolrClient.java:657) at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:589) at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:91) at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:243) at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:95) at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:844) at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1919) at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:182) at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:78) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:171) at org.apache.solr.core.SolrCore.execute(SolrCore.java:2420) at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:298) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) at org.eclipse.jetty.server.Server.handle(Server.java:534) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95) at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:
1) Thread[id=3211, name=qtp510760146-3211, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.lang.Throwable.fillInStackTrace(Native Method)
at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
at java.lang.Throwable.<init>(Throwable.java:250)
at java.lang.Exception.<init>(Exception.java:54)
at java.lang.InterruptedException.<init>(InterruptedException.java:57)
at java.lang.Object.wait(Native Method)
at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.waitForEmptyQueue(ConcurrentUpdateSolrClient.java:657)
at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:589)
at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:91)
at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:243)
at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:95)
at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:844)
at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1919)
at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:182)
at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:78)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:171)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:2420)
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:298)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([FBBC37A894D050A5]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=3211, name=qtp510760146-3211, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Object.wait(Native Method) at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.waitForEmptyQueue(ConcurrentUpdateSolrClient.java:657) at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:589) at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:91) at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:243) at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:95) at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:844) at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1919) at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:182) at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:78) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:171) at org.apache.solr.core.SolrCore.execute(SolrCore.java:2420) at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:298) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) at org.eclipse.jetty.server.Server.handle(Server.java:534) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95) at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=3211, name=qtp510760146-3211, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.lang.Object.wait(Native Method)
at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.waitForEmptyQueue(ConcurrentUpdateSolrClient.java:657)
at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:589)
at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:91)
at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:243)
at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:95)
at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:844)
at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1919)
at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:182)
at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:78)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:171)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:2420)
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:298)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([FBBC37A894D050A5]:0)
Build Log:
[...truncated 11676 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/init-core-data-001
[junit4] 2> 510755 INFO (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[FBBC37A894D050A5]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields
[junit4] 2> 510756 INFO (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[FBBC37A894D050A5]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 510756 INFO (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[FBBC37A894D050A5]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 510758 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 510758 INFO (Thread-959) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 510758 INFO (Thread-959) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 510858 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.ZkTestServer start zk server on port:42401
[junit4] 2> 510869 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 510871 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 510872 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 510873 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 510874 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 510875 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 510876 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 510877 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 510878 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 510879 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 510880 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 510947 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/control-001/cores/collection1
[junit4] 2> 510948 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 510950 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@290ab9c3{/,null,AVAILABLE}
[junit4] 2> 510954 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3fb5071a{HTTP/1.1,[http/1.1]}{127.0.0.1:60403}
[junit4] 2> 510954 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.Server Started @515010ms
[junit4] 2> 510954 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/tempDir-001/control/data, hostContext=/, hostPort=60403, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/control-001/cores}
[junit4] 2> 510954 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 510954 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 510954 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 510954 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 510954 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-04T19:01:26.547Z
[junit4] 2> 510957 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 510957 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/control-001/solr.xml
[junit4] 2> 510968 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42401/solr
[junit4] 2> 510985 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:60403_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 510988 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:60403_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60403_
[junit4] 2> 510989 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:60403_ ] o.a.s.c.Overseer Overseer (id=97560434203230212-127.0.0.1:60403_-n_0000000000) starting
[junit4] 2> 510994 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:60403_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60403_
[junit4] 2> 511000 INFO (zkCallback-395-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 511050 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:60403_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/control-001/cores
[junit4] 2> 511050 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:60403_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 511058 INFO (OverseerStateUpdate-97560434203230212-127.0.0.1:60403_-n_0000000000) [n:127.0.0.1:60403_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 512085 WARN (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 512086 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 512096 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 512143 WARN (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 512144 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 512159 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 512159 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/control-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/control-001/cores/collection1/data/]
[junit4] 2> 512160 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3d46f8f2
[junit4] 2> 512161 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@10ac6ad6
[junit4] 2> 512173 WARN (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 512210 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 512210 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 512211 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 512211 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 512212 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=24, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8567664016904397]
[junit4] 2> 512213 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1d5d7625[collection1] main]
[junit4] 2> 512214 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 512214 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 512214 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 512215 INFO (coreLoadExecutor-1336-thread-1-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1560966948777361408
[junit4] 2> 512220 INFO (coreZkRegister-1329-thread-1-processing-n:127.0.0.1:60403_ x:collection1 c:control_collection) [n:127.0.0.1:60403_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 512220 INFO (coreZkRegister-1329-thread-1-processing-n:127.0.0.1:60403_ x:collection1 c:control_collection) [n:127.0.0.1:60403_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 512220 INFO (coreZkRegister-1329-thread-1-processing-n:127.0.0.1:60403_ x:collection1 c:control_collection) [n:127.0.0.1:60403_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60403/collection1/
[junit4] 2> 512220 INFO (coreZkRegister-1329-thread-1-processing-n:127.0.0.1:60403_ x:collection1 c:control_collection) [n:127.0.0.1:60403_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 512220 INFO (coreZkRegister-1329-thread-1-processing-n:127.0.0.1:60403_ x:collection1 c:control_collection) [n:127.0.0.1:60403_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:60403/collection1/ has no replicas
[junit4] 2> 512220 INFO (coreZkRegister-1329-thread-1-processing-n:127.0.0.1:60403_ x:collection1 c:control_collection) [n:127.0.0.1:60403_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 512228 INFO (searcherExecutor-1337-thread-1-processing-n:127.0.0.1:60403_ x:collection1 c:control_collection) [n:127.0.0.1:60403_ c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1d5d7625[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 512242 INFO (coreZkRegister-1329-thread-1-processing-n:127.0.0.1:60403_ x:collection1 c:control_collection) [n:127.0.0.1:60403_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60403/collection1/ shard1
[junit4] 2> 512353 INFO (coreZkRegister-1329-thread-1-processing-n:127.0.0.1:60403_ x:collection1 c:control_collection) [n:127.0.0.1:60403_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 512609 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 512609 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42401/solr ready
[junit4] 2> 512609 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 512609 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 512680 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-1-001/cores/collection1
[junit4] 2> 512681 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-1-001
[junit4] 2> 512681 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 512683 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6ba667cc{/,null,AVAILABLE}
[junit4] 2> 512683 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@28517ea9{HTTP/1.1,[http/1.1]}{127.0.0.1:49811}
[junit4] 2> 512683 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.Server Started @516740ms
[junit4] 2> 512683 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=49811, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-1-001/cores}
[junit4] 2> 512684 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 512685 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 512685 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 512685 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 512685 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-04T19:01:28.278Z
[junit4] 2> 512687 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 512688 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-1-001/solr.xml
[junit4] 2> 512693 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42401/solr
[junit4] 2> 512708 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:49811_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 512709 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:49811_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 512711 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:49811_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49811_
[junit4] 2> 512714 INFO (zkCallback-399-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 512714 INFO (zkCallback-395-thread-4-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 512735 INFO (zkCallback-405-thread-2-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 512806 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:49811_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-1-001/cores
[junit4] 2> 512806 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:49811_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 512811 INFO (OverseerStateUpdate-97560434203230212-127.0.0.1:60403_-n_0000000000) [n:127.0.0.1:60403_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 512913 WARN (OverseerStateUpdate-97560434203230212-127.0.0.1:60403_-n_0000000000) [n:127.0.0.1:60403_ ] o.a.s.c.Overseer Bad version writing to ZK using compare-and-set, will force refresh cluster state: KeeperErrorCode = BadVersion for /collections/collection1/state.json
[junit4] 2> 512917 INFO (zkCallback-405-thread-2-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 512919 INFO (OverseerStateUpdate-97560434203230212-127.0.0.1:60403_-n_0000000000) [n:127.0.0.1:60403_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 512920 INFO (zkCallback-405-thread-2-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 513820 WARN (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 513821 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 513831 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 513885 WARN (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 513886 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 513901 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 513902 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-1-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 513902 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3d46f8f2
[junit4] 2> 513903 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@74f4c1b6
[junit4] 2> 513911 WARN (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 513936 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 513936 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 513937 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 513937 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 513938 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=24, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8567664016904397]
[junit4] 2> 513938 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@424b04d7[collection1] main]
[junit4] 2> 513939 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 513939 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 513939 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 513940 INFO (searcherExecutor-1348-thread-1-processing-n:127.0.0.1:49811_ x:collection1 c:collection1) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@424b04d7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 513940 INFO (coreLoadExecutor-1347-thread-1-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1560966950586155008
[junit4] 2> 513945 INFO (coreZkRegister-1342-thread-1-processing-n:127.0.0.1:49811_ x:collection1 c:collection1) [n:127.0.0.1:49811_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 513945 INFO (coreZkRegister-1342-thread-1-processing-n:127.0.0.1:49811_ x:collection1 c:collection1) [n:127.0.0.1:49811_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 513945 INFO (coreZkRegister-1342-thread-1-processing-n:127.0.0.1:49811_ x:collection1 c:collection1) [n:127.0.0.1:49811_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:49811/collection1/
[junit4] 2> 513945 INFO (coreZkRegister-1342-thread-1-processing-n:127.0.0.1:49811_ x:collection1 c:collection1) [n:127.0.0.1:49811_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 513945 INFO (coreZkRegister-1342-thread-1-processing-n:127.0.0.1:49811_ x:collection1 c:collection1) [n:127.0.0.1:49811_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:49811/collection1/ has no replicas
[junit4] 2> 513945 INFO (coreZkRegister-1342-thread-1-processing-n:127.0.0.1:49811_ x:collection1 c:collection1) [n:127.0.0.1:49811_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 513948 INFO (coreZkRegister-1342-thread-1-processing-n:127.0.0.1:49811_ x:collection1 c:collection1) [n:127.0.0.1:49811_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:49811/collection1/ shard2
[junit4] 2> 514049 INFO (zkCallback-405-thread-2-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 514099 INFO (coreZkRegister-1342-thread-1-processing-n:127.0.0.1:49811_ x:collection1 c:collection1) [n:127.0.0.1:49811_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 514201 INFO (zkCallback-405-thread-2-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 514428 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-2-001/cores/collection1
[junit4] 2> 514429 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-2-001
[junit4] 2> 514429 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 514430 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3615d08b{/,null,AVAILABLE}
[junit4] 2> 514430 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2dd0c880{HTTP/1.1,[http/1.1]}{127.0.0.1:57920}
[junit4] 2> 514430 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.Server Started @518487ms
[junit4] 2> 514435 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=57920, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-2-001/cores}
[junit4] 2> 514435 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 514435 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 514435 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 514435 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 514435 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-04T19:01:30.028Z
[junit4] 2> 514437 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 514437 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-2-001/solr.xml
[junit4] 2> 514443 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42401/solr
[junit4] 2> 514449 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:57920_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 514450 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:57920_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 514451 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:57920_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57920_
[junit4] 2> 514452 INFO (zkCallback-399-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 514452 INFO (zkCallback-405-thread-2-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 514453 INFO (zkCallback-395-thread-5-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 514465 INFO (zkCallback-411-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 514522 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:57920_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-2-001/cores
[junit4] 2> 514522 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:57920_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 514523 INFO (OverseerStateUpdate-97560434203230212-127.0.0.1:60403_-n_0000000000) [n:127.0.0.1:60403_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard3
[junit4] 2> 514625 INFO (zkCallback-411-thread-2-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 514625 INFO (zkCallback-405-thread-2-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 515534 WARN (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 515535 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 515557 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 515603 WARN (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 515604 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 515616 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 515617 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-2-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 515617 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3d46f8f2
[junit4] 2> 515618 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@5d576112
[junit4] 2> 515626 WARN (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 515652 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 515652 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 515653 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 515653 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 515654 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=24, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8567664016904397]
[junit4] 2> 515654 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7add8aa8[collection1] main]
[junit4] 2> 515658 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 515658 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 515658 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 515661 INFO (searcherExecutor-1359-thread-1-processing-n:127.0.0.1:57920_ x:collection1 c:collection1) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7add8aa8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 515662 INFO (coreLoadExecutor-1358-thread-1-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1560966952391802880
[junit4] 2> 515667 INFO (coreZkRegister-1353-thread-1-processing-n:127.0.0.1:57920_ x:collection1 c:collection1) [n:127.0.0.1:57920_ c:collection1 s:shard3 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 515668 INFO (coreZkRegister-1353-thread-1-processing-n:127.0.0.1:57920_ x:collection1 c:collection1) [n:127.0.0.1:57920_ c:collection1 s:shard3 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 515668 INFO (coreZkRegister-1353-thread-1-processing-n:127.0.0.1:57920_ x:collection1 c:collection1) [n:127.0.0.1:57920_ c:collection1 s:shard3 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:57920/collection1/
[junit4] 2> 515668 INFO (coreZkRegister-1353-thread-1-processing-n:127.0.0.1:57920_ x:collection1 c:collection1) [n:127.0.0.1:57920_ c:collection1 s:shard3 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 515668 INFO (coreZkRegister-1353-thread-1-processing-n:127.0.0.1:57920_ x:collection1 c:collection1) [n:127.0.0.1:57920_ c:collection1 s:shard3 r:core_node2 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:57920/collection1/ has no replicas
[junit4] 2> 515668 INFO (coreZkRegister-1353-thread-1-processing-n:127.0.0.1:57920_ x:collection1 c:collection1) [n:127.0.0.1:57920_ c:collection1 s:shard3 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 515670 INFO (coreZkRegister-1353-thread-1-processing-n:127.0.0.1:57920_ x:collection1 c:collection1) [n:127.0.0.1:57920_ c:collection1 s:shard3 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:57920/collection1/ shard3
[junit4] 2> 515771 INFO (zkCallback-405-thread-2-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 515773 INFO (zkCallback-411-thread-2-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 515823 INFO (coreZkRegister-1353-thread-1-processing-n:127.0.0.1:57920_ x:collection1 c:collection1) [n:127.0.0.1:57920_ c:collection1 s:shard3 r:core_node2 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 515927 INFO (zkCallback-405-thread-2-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 515927 INFO (zkCallback-411-thread-2-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 516138 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-3-001/cores/collection1
[junit4] 2> 516139 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-3-001
[junit4] 2> 516139 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 516140 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@646ab145{/,null,AVAILABLE}
[junit4] 2> 516141 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@35643059{HTTP/1.1,[http/1.1]}{127.0.0.1:57955}
[junit4] 2> 516141 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.Server Started @520198ms
[junit4] 2> 516141 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=57955, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-3-001/cores}
[junit4] 2> 516141 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 516143 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 516143 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 516143 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 516143 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-04T19:01:31.736Z
[junit4] 2> 516146 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 516146 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-3-001/solr.xml
[junit4] 2> 516155 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42401/solr
[junit4] 2> 516175 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:57955_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 516177 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:57955_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 516178 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:57955_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57955_
[junit4] 2> 516179 INFO (zkCallback-395-thread-5-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 516180 INFO (zkCallback-405-thread-2-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 516180 INFO (zkCallback-399-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 516179 INFO (zkCallback-411-thread-2-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 516190 INFO (zkCallback-417-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 516265 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:57955_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-3-001/cores
[junit4] 2> 516266 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:57955_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 516272 INFO (OverseerStateUpdate-97560434203230212-127.0.0.1:60403_-n_0000000000) [n:127.0.0.1:60403_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard4
[junit4] 2> 516374 INFO (zkCallback-405-thread-2-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 516374 INFO (zkCallback-411-thread-2-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 516374 INFO (zkCallback-417-thread-2-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 517281 WARN (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 517282 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 517292 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 517383 WARN (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 517384 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 517417 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 517417 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-3-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 517418 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3d46f8f2
[junit4] 2> 517419 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@6e8c31af
[junit4] 2> 517426 WARN (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 517452 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 517452 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 517453 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 517453 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 517454 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=24, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8567664016904397]
[junit4] 2> 517454 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3d0b571[collection1] main]
[junit4] 2> 517455 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 517455 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 517455 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 517456 INFO (coreLoadExecutor-1369-thread-1-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1560966954272948224
[junit4] 2> 517456 INFO (searcherExecutor-1370-thread-1-processing-n:127.0.0.1:57955_ x:collection1 c:collection1) [n:127.0.0.1:57955_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3d0b571[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 517470 INFO (coreZkRegister-1364-thread-1-processing-n:127.0.0.1:57955_ x:collection1 c:collection1) [n:127.0.0.1:57955_ c:collection1 s:shard4 r:core_node3 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 517470 INFO (coreZkRegister-1364-thread-1-processing-n:127.0.0.1:57955_ x:collection1 c:collection1) [n:127.0.0.1:57955_ c:collection1 s:shard4 r:core_node3 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 517470 INFO (coreZkRegister-1364-thread-1-processing-n:127.0.0.1:57955_ x:collection1 c:collection1) [n:127.0.0.1:57955_ c:collection1 s:shard4 r:core_node3 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:57955/collection1/
[junit4] 2> 517470 INFO (coreZkRegister-1364-thread-1-processing-n:127.0.0.1:57955_ x:collection1 c:collection1) [n:127.0.0.1:57955_ c:collection1 s:shard4 r:core_node3 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 517470 INFO (coreZkRegister-1364-thread-1-processing-n:127.0.0.1:57955_ x:collection1 c:collection1) [n:127.0.0.1:57955_ c:collection1 s:shard4 r:core_node3 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:57955/collection1/ has no replicas
[junit4] 2> 517470 INFO (coreZkRegister-1364-thread-1-processing-n:127.0.0.1:57955_ x:collection1 c:collection1) [n:127.0.0.1:57955_ c:collection1 s:shard4 r:core_node3 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 517473 INFO (coreZkRegister-1364-thread-1-processing-n:127.0.0.1:57955_ x:collection1 c:collection1) [n:127.0.0.1:57955_ c:collection1 s:shard4 r:core_node3 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:57955/collection1/ shard4
[junit4] 2> 517574 INFO (zkCallback-405-thread-2-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 517574 INFO (zkCallback-417-thread-2-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 517574 INFO (zkCallback-411-thread-2-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 517624 INFO (coreZkRegister-1364-thread-1-processing-n:127.0.0.1:57955_ x:collection1 c:collection1) [n:127.0.0.1:57955_ c:collection1 s:shard4 r:core_node3 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 517727 INFO (zkCallback-405-thread-2-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 517731 INFO (zkCallback-411-thread-2-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 517731 INFO (zkCallback-417-thread-2-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 517930 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-4-001/cores/collection1
[junit4] 2> 517931 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-4-001
[junit4] 2> 517933 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 517934 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2bd73fad{/,null,AVAILABLE}
[junit4] 2> 517935 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3b7cbe76{HTTP/1.1,[http/1.1]}{127.0.0.1:34490}
[junit4] 2> 517935 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.e.j.s.Server Started @521991ms
[junit4] 2> 517935 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/, hostPort=34490, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-4-001/cores}
[junit4] 2> 517935 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 517942 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 517942 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 517942 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 517942 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-04T19:01:33.535Z
[junit4] 2> 517976 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 517976 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_FBBC37A894D050A5-001/shard-4-001/solr.xml
[junit4] 2> 517982 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42401/solr
[junit4] 2> 518013 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:34490_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 518015 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:34490_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 518016 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:34490_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34490_
[junit4] 2> 518023 INFO (zkCallback-405-thread-2-processing-n:127.0.0.1:49811_) [n:127.0.0.1:49811_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 518024 INFO (zkCallback-411-thread-2-processing-n:127.0.0.1:57920_) [n:127.0.0.1:57920_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 518024 INFO (zkCallback-417-thread-2-processing-n:127.0.0.1:57955_) [n:127.0.0.1:57955_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 518024 INFO (zkCallback-395-thread-4-processing-n:127.0.0.1:60403_) [n:127.0.0.1:60403_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 518024 INFO (zkCallback-399-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 518053 INFO (zkCallback-423-thread-1-processing-n:127.0.0.1:34490_) [n:127.0.0.1:34490_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 518111 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[FBBC37A894D050A5]) [n:127.0.0.1:34490_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Nig
[...truncated too long message...]
ingIsSafeTest:
[junit4] > 1) Thread[id=3211, name=qtp510760146-3211, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.lang.Throwable.fillInStackTrace(Native Method)
[junit4] > at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
[junit4] > at java.lang.Throwable.<init>(Throwable.java:250)
[junit4] > at java.lang.Exception.<init>(Exception.java:54)
[junit4] > at java.lang.InterruptedException.<init>(InterruptedException.java:57)
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.waitForEmptyQueue(ConcurrentUpdateSolrClient.java:657)
[junit4] > at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:589)
[junit4] > at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:91)
[junit4] > at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:243)
[junit4] > at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:95)
[junit4] > at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:844)
[junit4] > at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1919)
[junit4] > at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:182)
[junit4] > at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:78)
[junit4] > at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:171)
[junit4] > at org.apache.solr.core.SolrCore.execute(SolrCore.java:2420)
[junit4] > at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722)
[junit4] > at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528)
[junit4] > at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
[junit4] > at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:298)
[junit4] > at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] > at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] > at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] > at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] > at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] > at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] > at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] > at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] > at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] > at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] > at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
[junit4] > at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] > at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] > at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] > at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] > at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
[junit4] > at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
[junit4] > at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([FBBC37A894D050A5]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=3211, name=qtp510760146-3211, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.waitForEmptyQueue(ConcurrentUpdateSolrClient.java:657)
[junit4] > at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:589)
[junit4] > at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:91)
[junit4] > at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:243)
[junit4] > at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:95)
[junit4] > at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:844)
[junit4] > at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1919)
[junit4] > at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:182)
[junit4] > at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:78)
[junit4] > at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:171)
[junit4] > at org.apache.solr.core.SolrCore.execute(SolrCore.java:2420)
[junit4] > at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:722)
[junit4] > at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:528)
[junit4] > at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
[junit4] > at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:298)
[junit4] > at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] > at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] > at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] > at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] > at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] > at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] > at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] > at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] > at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] > at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] > at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
[junit4] > at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] > at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] > at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] > at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] > at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
[junit4] > at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
[junit4] > at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([FBBC37A894D050A5]:0)
[junit4] Completed [117/693 (1!)] on J1 in 124.59s, 1 test, 2 errors <<< FAILURES!
[...truncated 64349 lines...]