You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2017/03/05 18:02:23 UTC
[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1255 - Still
Unstable
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1255/
2 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.HdfsTransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130) at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:104) at sun.reflect.GeneratedConstructorAccessor157.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:759) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:821) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1071) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:936) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:829) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:937) at org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:572) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.HdfsTransactionLog
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:104)
at sun.reflect.GeneratedConstructorAccessor157.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:759)
at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:821)
at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1071)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:936)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:829)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:937)
at org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:572)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([A5A0B1F6AC0C365C]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
at sun.reflect.GeneratedMethodAccessor80.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
FAILED: org.apache.solr.handler.admin.SegmentsInfoRequestHandlerTest.testSegmentInfosVersion
Error Message:
Exception during query
Stack Trace:
java.lang.RuntimeException: Exception during query
at __randomizedtesting.SeedInfo.seed([A5A0B1F6AC0C365C:5D7E241ED460E70F]:0)
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:919)
at org.apache.solr.handler.admin.SegmentsInfoRequestHandlerTest.testSegmentInfosVersion(SegmentsInfoRequestHandlerTest.java:70)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=2=count(//lst[@name='segments']/lst/str[@name='version'][.='7.0.0'])
xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">3</int></lst><lst name="segments"><lst name="_0"><str name="name">_0</str><int name="delCount">1</int><long name="sizeInBytes">1732</long><int name="size">4</int><date name="age">2017-03-05T17:35:43.518Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst><lst name="_1"><str name="name">_1</str><int name="delCount">0</int><long name="sizeInBytes">1581</long><int name="size">1</int><date name="age">2017-03-05T17:35:43.520Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst><lst name="_2"><str name="name">_2</str><int name="delCount">0</int><long name="sizeInBytes">1665</long><int name="size">4</int><date name="age">2017-03-05T17:35:43.527Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst><lst name="_3"><str name="name">_3</str><int name="delCount">0</int><long name="sizeInBytes">1581</long><int name="size">1</int><date name="age">2017-03-05T17:35:43.528Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst></lst>
</response>
request was:qt=/admin/segments&wt=xml
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:912)
... 40 more
Build Log:
[...truncated 13108 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A5A0B1F6AC0C365C-001/init-core-data-001
[junit4] 2> 3571238 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields
[junit4] 2> 3571239 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 3571239 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A5A0B1F6AC0C365C-001/tempDir-001
[junit4] 2> 3571239 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3571240 INFO (Thread-79513) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3571240 INFO (Thread-79513) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3571340 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:55857
[junit4] 2> 3571346 INFO (jetty-launcher-33238-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3571346 INFO (jetty-launcher-33238-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3571349 INFO (jetty-launcher-33238-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@54fd3937{/solr,null,AVAILABLE}
[junit4] 2> 3571350 INFO (jetty-launcher-33238-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@cd71c91{HTTP/1.1,[http/1.1]}{127.0.0.1:59336}
[junit4] 2> 3571350 INFO (jetty-launcher-33238-thread-2) [ ] o.e.j.s.Server Started @3573815ms
[junit4] 2> 3571350 INFO (jetty-launcher-33238-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59336}
[junit4] 2> 3571350 ERROR (jetty-launcher-33238-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3571351 INFO (jetty-launcher-33238-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 3571351 INFO (jetty-launcher-33238-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3571351 INFO (jetty-launcher-33238-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3571351 INFO (jetty-launcher-33238-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-05T17:33:15.615Z
[junit4] 2> 3571351 INFO (jetty-launcher-33238-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5721e3be{/solr,null,AVAILABLE}
[junit4] 2> 3571352 INFO (jetty-launcher-33238-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@78f49321{HTTP/1.1,[http/1.1]}{127.0.0.1:33936}
[junit4] 2> 3571352 INFO (jetty-launcher-33238-thread-1) [ ] o.e.j.s.Server Started @3573817ms
[junit4] 2> 3571352 INFO (jetty-launcher-33238-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33936}
[junit4] 2> 3571352 ERROR (jetty-launcher-33238-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3571352 INFO (jetty-launcher-33238-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 3571352 INFO (jetty-launcher-33238-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3571352 INFO (jetty-launcher-33238-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3571352 INFO (jetty-launcher-33238-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-03-05T17:33:15.616Z
[junit4] 2> 3571353 INFO (jetty-launcher-33238-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3571354 INFO (jetty-launcher-33238-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3571359 INFO (jetty-launcher-33238-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55857/solr
[junit4] 2> 3571359 INFO (jetty-launcher-33238-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55857/solr
[junit4] 2> 3571374 INFO (jetty-launcher-33238-thread-2) [n:127.0.0.1:59336_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3571374 INFO (jetty-launcher-33238-thread-1) [n:127.0.0.1:33936_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3571374 INFO (jetty-launcher-33238-thread-2) [n:127.0.0.1:59336_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59336_solr
[junit4] 2> 3571375 INFO (jetty-launcher-33238-thread-1) [n:127.0.0.1:33936_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33936_solr
[junit4] 2> 3571375 INFO (jetty-launcher-33238-thread-2) [n:127.0.0.1:59336_solr ] o.a.s.c.Overseer Overseer (id=97565749772550150-127.0.0.1:59336_solr-n_0000000000) starting
[junit4] 2> 3571387 INFO (zkCallback-33249-thread-1-processing-n:127.0.0.1:33936_solr) [n:127.0.0.1:33936_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3571387 INFO (zkCallback-33250-thread-1-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3571400 INFO (jetty-launcher-33238-thread-2) [n:127.0.0.1:59336_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59336_solr
[junit4] 2> 3571401 INFO (zkCallback-33249-thread-1-processing-n:127.0.0.1:33936_solr) [n:127.0.0.1:33936_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3571407 INFO (zkCallback-33250-thread-1-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3571424 INFO (jetty-launcher-33238-thread-1) [n:127.0.0.1:33936_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A5A0B1F6AC0C365C-001/tempDir-001/node1/.
[junit4] 2> 3571481 INFO (jetty-launcher-33238-thread-2) [n:127.0.0.1:59336_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A5A0B1F6AC0C365C-001/tempDir-001/node2/.
[junit4] 2> 3572239 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3572239 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55857/solr ready
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 3572267 WARN (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 3572277 WARN (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3572278 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3572288 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_51840_hdfs____46ljya/webapp
[junit4] 2> 3572621 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:51840
[junit4] 2> 3572673 WARN (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3572674 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3572683 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_48998_datanode____.f46osl/webapp
[junit4] 2> 3573077 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:48998
[junit4] 2> 3573127 WARN (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3573128 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3573137 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_53708_datanode____3u2q2w/webapp
[junit4] 2> 3573208 INFO (IPC Server handler 3 on 50422) [ ] BlockStateChange BLOCK* processReport: from storage DS-7e37e0d8-8022-4162-89a0-512a21232c5f node DatanodeRegistration(127.0.0.1:60707, datanodeUuid=5a741cd7-a89a-4458-a888-e844cd55c2d5, infoPort=43443, infoSecurePort=0, ipcPort=50407, storageInfo=lv=-56;cid=testClusterID;nsid=898209660;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3573208 INFO (IPC Server handler 3 on 50422) [ ] BlockStateChange BLOCK* processReport: from storage DS-a74a72c2-9aab-47bd-ab16-a688cef64514 node DatanodeRegistration(127.0.0.1:60707, datanodeUuid=5a741cd7-a89a-4458-a888-e844cd55c2d5, infoPort=43443, infoSecurePort=0, ipcPort=50407, storageInfo=lv=-56;cid=testClusterID;nsid=898209660;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3573540 INFO (SUITE-HdfsRecoveryZkTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:53708
[junit4] 2> 3573612 INFO (IPC Server handler 8 on 50422) [ ] BlockStateChange BLOCK* processReport: from storage DS-3fac77ef-9841-4d7b-bd84-a6cb027ce3ad node DatanodeRegistration(127.0.0.1:41670, datanodeUuid=b958c4e3-95ea-4d33-a066-624181e0aef7, infoPort=32953, infoSecurePort=0, ipcPort=32961, storageInfo=lv=-56;cid=testClusterID;nsid=898209660;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3573612 INFO (IPC Server handler 8 on 50422) [ ] BlockStateChange BLOCK* processReport: from storage DS-e2131f17-47c5-481d-b327-961042eaabcc node DatanodeRegistration(127.0.0.1:41670, datanodeUuid=b958c4e3-95ea-4d33-a066-624181e0aef7, infoPort=32953, infoSecurePort=0, ipcPort=32961, storageInfo=lv=-56;cid=testClusterID;nsid=898209660;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3573746 INFO (TEST-HdfsRecoveryZkTest.test-seed#[A5A0B1F6AC0C365C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 3573747 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3573749 INFO (OverseerThreadFactory-15140-thread-1-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 3573855 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 3573855 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 3573961 INFO (zkCallback-33249-thread-1-processing-n:127.0.0.1:33936_solr) [n:127.0.0.1:33936_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3573961 INFO (zkCallback-33250-thread-3-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3574864 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 3574864 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 3574870 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema [recoverytest_shard1_replica2] Schema name=minimal
[junit4] 2> 3574870 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema [recoverytest_shard1_replica1] Schema name=minimal
[junit4] 2> 3574872 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3574872 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3574873 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica1' using configuration from collection recoverytest
[junit4] 2> 3574873 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica2' using configuration from collection recoverytest
[junit4] 2> 3574878 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:50422/data
[junit4] 2> 3574878 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3574878 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [[recoverytest_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A5A0B1F6AC0C365C-001/tempDir-001/node2/recoverytest_shard1_replica1], dataDir=[hdfs://localhost:50422/data/recoverytest/core_node2/data/]
[junit4] 2> 3574878 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:50422/data
[junit4] 2> 3574878 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3574878 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6fd8959d
[junit4] 2> 3574878 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [[recoverytest_shard1_replica2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_A5A0B1F6AC0C365C-001/tempDir-001/node1/recoverytest_shard1_replica2], dataDir=[hdfs://localhost:50422/data/recoverytest/core_node1/data/]
[junit4] 2> 3574878 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6fd8959d
[junit4] 2> 3574878 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:50422/data/recoverytest/core_node2/data/snapshot_metadata
[junit4] 2> 3574879 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:50422/data/recoverytest/core_node1/data/snapshot_metadata
[junit4] 2> 3574885 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3574885 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3574885 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3574885 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3574887 WARN (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 3574887 WARN (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 3579893 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3579894 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3579895 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:50422/data/recoverytest/core_node2/data
[junit4] 2> 3579895 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:50422/data/recoverytest/core_node1/data
[junit4] 2> 3579909 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:50422/data/recoverytest/core_node2/data/index
[junit4] 2> 3579919 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:50422/data/recoverytest/core_node1/data/index
[junit4] 2> 3579920 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3579920 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3579929 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3579929 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3579929 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3579953 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3579978 INFO (IPC Server handler 9 on 50422) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41670 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-7e37e0d8-8022-4162-89a0-512a21232c5f:NORMAL:127.0.0.1:60707|RBW], ReplicaUC[[DISK]DS-3fac77ef-9841-4d7b-bd84-a6cb027ce3ad:NORMAL:127.0.0.1:41670|FINALIZED]]} size 0
[junit4] 2> 3579979 INFO (IPC Server handler 0 on 50422) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:60707 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-7e37e0d8-8022-4162-89a0-512a21232c5f:NORMAL:127.0.0.1:60707|RBW], ReplicaUC[[DISK]DS-3fac77ef-9841-4d7b-bd84-a6cb027ce3ad:NORMAL:127.0.0.1:41670|FINALIZED]]} size 0
[junit4] 2> 3579982 INFO (IPC Server handler 5 on 50422) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41670 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-e2131f17-47c5-481d-b327-961042eaabcc:NORMAL:127.0.0.1:41670|RBW], ReplicaUC[[DISK]DS-a74a72c2-9aab-47bd-ab16-a688cef64514:NORMAL:127.0.0.1:60707|RBW]]} size 0
[junit4] 2> 3579982 INFO (IPC Server handler 6 on 50422) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:60707 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-e2131f17-47c5-481d-b327-961042eaabcc:NORMAL:127.0.0.1:41670|RBW], ReplicaUC[[DISK]DS-a74a72c2-9aab-47bd-ab16-a688cef64514:NORMAL:127.0.0.1:60707|RBW]]} size 0
[junit4] 2> 3580027 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 3580027 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3580027 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 3580035 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3580035 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3580041 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 3580041 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3580041 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 3580049 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3580049 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3580052 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2d9fd695[recoverytest_shard1_replica1] main]
[junit4] 2> 3580053 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 3580054 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 3580054 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3580059 INFO (searcherExecutor-15145-thread-1-processing-n:127.0.0.1:59336_solr x:recoverytest_shard1_replica1 s:shard1 c:recoverytest) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [recoverytest_shard1_replica1] Registered new searcher Searcher@2d9fd695[recoverytest_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3580062 WARN (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:58117/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:58117 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3580062 WARN (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:58117/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:58117 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3580063 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1561052005612388352
[junit4] 2> 3580066 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@732ca366[recoverytest_shard1_replica2] main]
[junit4] 2> 3580067 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 3580067 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 3580067 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3580068 INFO (searcherExecutor-15146-thread-1-processing-n:127.0.0.1:33936_solr x:recoverytest_shard1_replica2 s:shard1 c:recoverytest) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [recoverytest_shard1_replica2] Registered new searcher Searcher@732ca366[recoverytest_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3580070 INFO (zkCallback-33250-thread-1-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3580070 WARN (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:58117/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:58117 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3580071 WARN (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.h.HdfsLocalityReporter Could not retrieve locality information for hdfs://localhost:58117/solr3 due to exception: java.net.ConnectException: Call From lucene1-us-west/10.41.0.5 to localhost:58117 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused
[junit4] 2> 3580071 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1561052005620776960
[junit4] 2> 3580071 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 3580073 INFO (zkCallback-33249-thread-1-processing-n:127.0.0.1:33936_solr) [n:127.0.0.1:33936_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3580591 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3580592 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3580592 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/
[junit4] 2> 3580592 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica1 url=http://127.0.0.1:59336/solr START replicas=[http://127.0.0.1:33936/solr/recoverytest_shard1_replica2/] nUpdates=100
[junit4] 2> 3580592 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica1 url=http://127.0.0.1:59336/solr DONE. We have no versions. sync failed.
[junit4] 2> 3580605 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.S.Request [recoverytest_shard1_replica2] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 3580606 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 3580606 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 3580606 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3580609 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/ shard1
[junit4] 2> 3580713 INFO (zkCallback-33249-thread-1-processing-n:127.0.0.1:33936_solr) [n:127.0.0.1:33936_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3580713 INFO (zkCallback-33250-thread-3-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3580759 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3580762 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=6906
[junit4] 2> 3580865 INFO (zkCallback-33249-thread-1-processing-n:127.0.0.1:33936_solr) [n:127.0.0.1:33936_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3580865 INFO (zkCallback-33250-thread-2-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3581076 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=7221
[junit4] 2> 3581079 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3581179 INFO (zkCallback-33249-thread-1-processing-n:127.0.0.1:33936_solr) [n:127.0.0.1:33936_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3581179 INFO (zkCallback-33250-thread-3-processing-n:127.0.0.1:59336_solr) [n:127.0.0.1:59336_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3581753 INFO (OverseerCollectionConfigSetProcessor-97565749772550150-127.0.0.1:59336_solr-n_0000000000) [n:127.0.0.1:59336_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3582080 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=8332
[junit4] 2> 3582081 INFO (TEST-HdfsRecoveryZkTest.test-seed#[A5A0B1F6AC0C365C]) [ ] o.a.s.c.RecoveryZkTest Indexing 12000 documents
[junit4] 2> 3582090 DEBUG (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.HdfsTransactionLog Opening new tlog hdfs tlog{file=hdfs://localhost:50422/data/recoverytest/core_node2/data/tlog/tlog.0000000000000000000 refcount=1}
[junit4] 2> 3582096 DEBUG (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.HdfsTransactionLog Opening new tlog hdfs tlog{file=hdfs://localhost:50422/data/recoverytest/core_node1/data/tlog/tlog.0000000000000000000 refcount=1}
[junit4] 2> 3582100 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-0 (1561052007732609024)]} 0 7
[junit4] 2> 3582100 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 (1561052007732609024)]} 0 15
[junit4] 2> 3582101 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-0 (1561052007731560448)]} 0 9
[junit4] 2> 3582101 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 (1561052007731560448)]} 0 17
[junit4] 2> 3582103 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-0 (-1561052007749386240)]} 0 1
[junit4] 2> 3582104 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-0 (-1561052007749386240)]} 0 2
[junit4] 2> 3582104 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-0 (-1561052007750434816)]} 0 0
[junit4] 2> 3582104 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-0 (-1561052007750434816)]} 0 2
[junit4] 2> 3582105 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-1 (1561052007752531968)]} 0 0
[junit4] 2> 3582106 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 (1561052007752531968)]} 0 1
[junit4] 2> 3582106 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-1 (1561052007753580544)]} 0 0
[junit4] 2> 3582107 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 (1561052007753580544)]} 0 2
[junit4] 2> 3582107 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-2 (1561052007754629120)]} 0 0
[junit4] 2> 3582107 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 (1561052007754629120)]} 0 1
[junit4] 2> 3582109 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-2 (1561052007755677696)]} 0 0
[junit4] 2> 3582109 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 (1561052007755677696)]} 0 1
[junit4] 2> 3582109 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-1 (-1561052007756726272)]} 0 0
[junit4] 2> 3582109 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-1 (-1561052007756726272)]} 0 1
[junit4] 2> 3582111 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-1 (-1561052007758823424)]} 0 1
[junit4] 2> 3582111 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-3 (1561052007758823425)]} 0 0
[junit4] 2> 3582111 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-1 (-1561052007758823424)]} 0 1
[junit4] 2> 3582111 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 (1561052007758823425)]} 0 1
[junit4] 2> 3582113 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-4 (1561052007760920577)]} 0 0
[junit4] 2> 3582113 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 (1561052007760920577)]} 0 1
[junit4] 2> 3582113 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-3 (1561052007760920576)]} 0 0
[junit4] 2> 3582114 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 (1561052007760920576)]} 0 1
[junit4] 2> 3582115 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-5 (1561052007763017728)]} 0 0
[junit4] 2> 3582115 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 (1561052007763017728)]} 0 1
[junit4] 2> 3582119 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-4 (1561052007763017729)]} 0 3
[junit4] 2> 3582119 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 (1561052007763017729)]} 0 4
[junit4] 2> 3582119 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-3 (-1561052007765114880)]} 0 3
[junit4] 2> 3582119 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-3 (-1561052007765114880)]} 0 3
[junit4] 2> 3582120 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-6 (1561052007769309184)]} 0 0
[junit4] 2> 3582120 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 (1561052007769309184)]} 0 0
[junit4] 2> 3582120 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-5 (1561052007768260608)]} 0 0
[junit4] 2> 3582121 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 (1561052007768260608)]} 0 1
[junit4] 2> 3582122 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-4 (-1561052007770357760)]} 0 0
[junit4] 2> 3582122 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-4 (-1561052007770357760)]} 0 0
[junit4] 2> 3582122 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-3 (-1561052007770357761)]} 0 1
[junit4] 2> 3582123 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-3 (-1561052007770357761)]} 0 1
[junit4] 2> 3582123 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-7 (1561052007771406336)]} 0 0
[junit4] 2> 3582124 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 (1561052007771406336)]} 0 1
[junit4] 2> 3582124 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-6 (1561052007772454912)]} 0 0
[junit4] 2> 3582124 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-6 (1561052007772454912)]} 0 1
[junit4] 2> 3582125 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-8 (1561052007773503488)]} 0 0
[junit4] 2> 3582125 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 (1561052007773503488)]} 0 1
[junit4] 2> 3582126 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-4 (-1561052007774552064)]} 0 0
[junit4] 2> 3582126 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-4 (-1561052007774552064)]} 0 1
[junit4] 2> 3582129 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-6 (-1561052007775600640)]} 0 1
[junit4] 2> 3582129 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-6 (-1561052007775600640)]} 0 3
[junit4] 2> 3582129 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-7 (1561052007776649216)]} 0 1
[junit4] 2> 3582129 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-7 (1561052007776649216)]} 0 2
[junit4] 2> 3582131 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-9 (1561052007778746368)]} 0 0
[junit4] 2> 3582131 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 (1561052007778746368)]} 0 1
[junit4] 2> 3582131 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-8 (1561052007779794944)]} 0 0
[junit4] 2> 3582132 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-8 (1561052007779794944)]} 0 2
[junit4] 2> 3582132 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-8 (-1561052007780843520)]} 0 0
[junit4] 2> 3582133 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-8 (-1561052007780843520)]} 0 1
[junit4] 2> 3582133 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-6 (-1561052007781892096)]} 0 0
[junit4] 2> 3582133 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-6 (-1561052007781892096)]} 0 1
[junit4] 2> 3582135 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-10 (1561052007782940672)]} 0 2
[junit4] 2> 3582136 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 (1561052007782940672)]} 0 2
[junit4] 2> 3582138 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-9 (-1561052007786086400)]} 0 0
[junit4] 2> 3582138 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-9 (-1561052007786086400)]} 0 1
[junit4] 2> 3582138 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-9 (1561052007786086401)]} 0 1
[junit4] 2> 3582138 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-9 (1561052007786086401)]} 0 1
[junit4] 2> 3582139 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-11 (1561052007788183552)]} 0 0
[junit4] 2> 3582139 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-8 (-1561052007789232128)]} 0 0
[junit4] 2> 3582139 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-11 (1561052007788183552)]} 0 1
[junit4] 2> 3582139 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-8 (-1561052007789232128)]} 0 0
[junit4] 2> 3582141 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-10 (-1561052007790280704)]} 0 0
[junit4] 2> 3582141 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-10 (-1561052007790280704)]} 0 1
[junit4] 2> 3582141 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-10 (1561052007790280705)]} 0 0
[junit4] 2> 3582142 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-10 (1561052007790280705)]} 0 1
[junit4] 2> 3582143 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-12 (1561052007791329280)]} 0 0
[junit4] 2> 3582143 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-12 (1561052007791329280)]} 0 1
[junit4] 2> 3582143 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-9 (-1561052007792377856)]} 0 0
[junit4] 2> 3582143 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-9 (-1561052007792377856)]} 0 1
[junit4] 2> 3582144 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-12 (-1561052007793426432)]} 0 0
[junit4] 2> 3582145 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-12 (-1561052007793426432)]} 0 1
[junit4] 2> 3582145 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-11 (1561052007794475008)]} 0 0
[junit4] 2> 3582145 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-11 (1561052007794475008)]} 0 1
[junit4] 2> 3582146 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-13 (1561052007795523585)]} 0 0
[junit4] 2> 3582146 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-13 (1561052007795523585)]} 0 1
[junit4] 2> 3582147 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-10 (-1561052007795523584)]} 0 0
[junit4] 2> 3582147 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-10 (-1561052007795523584)]} 0 1
[junit4] 2> 3582148 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-13 (-1561052007797620736)]} 0 0
[junit4] 2> 3582148 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-13 (-1561052007797620736)]} 0 1
[junit4] 2> 3582148 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-12 (1561052007797620737)]} 0 0
[junit4] 2> 3582149 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-12 (1561052007797620737)]} 0 1
[junit4] 2> 3582150 INFO (qtp964039117-204414) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-14 (1561052007799717888)]} 0 0
[junit4] 2> 3582150 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-14 (1561052007799717888)]} 0 1
[junit4] 2> 3582150 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-12 (-1561052007799717889)]} 0 0
[junit4] 2> 3582150 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-12 (-1561052007799717889)]} 0 1
[junit4] 2> 3582152 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-15 (1561052007801815040)]} 0 0
[junit4] 2> 3582152 INFO (qtp136419885-204417) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-15 (1561052007801815040)]} 0 1
[junit4] 2> 3582152 INFO (qtp964039117-204416) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-13 (1561052007801815041)]} 0 0
[junit4] 2> 3582153 INFO (qtp136419885-204420) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-13 (1561052007801815041)]} 0 1
[junit4] 2> 3582154 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-14 (-1561052007803912192)]} 0 0
[junit4] 2> 3582154 INFO (qtp136419885-204418) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-14 (-1561052007803912192)]} 0 1
[junit4] 2> 3582154 INFO (qtp964039117-204415) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-13 (-1561052007803912193)]} 0 0
[junit4] 2> 3582154 INFO (qtp136419885-204422) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-13 (-1561052007803912193)]} 0 1
[junit4] 2> 3582155 INFO (qtp964039117-204407) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-16 (1561052007804960768)]} 0 0
[junit4] 2> 3582156 INFO (qtp136419885-204408) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-16 (1561052007804960768)]} 0 1
[junit4] 2> 3582156 INFO (qtp964039117-204411) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:59336/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-14 (1561052007806009344)]} 0 0
[junit4] 2> 3582157 INFO (qtp136419885-204421) [n:127.0.0.1:59336_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-14 (1561052007806009344)]} 0 1
[junit4] 2> 3582157 INFO (qtp964039117-204409) [n:127.0.0.1:33936_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.Log
[...truncated too long message...]
h.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 3718928 INFO (TEST-SegmentsInfoRequestHandlerTest.testSegmentInfosVersion-seed#[A5A0B1F6AC0C365C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSegmentInfosVersion
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=SegmentsInfoRequestHandlerTest -Dtests.method=testSegmentInfosVersion -Dtests.seed=A5A0B1F6AC0C365C -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=zh -Dtests.timezone=Brazil/Acre -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.01s J1 | SegmentsInfoRequestHandlerTest.testSegmentInfosVersion <<<
[junit4] > Throwable #1: java.lang.RuntimeException: Exception during query
[junit4] > at __randomizedtesting.SeedInfo.seed([A5A0B1F6AC0C365C:5D7E241ED460E70F]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:919)
[junit4] > at org.apache.solr.handler.admin.SegmentsInfoRequestHandlerTest.testSegmentInfosVersion(SegmentsInfoRequestHandlerTest.java:70)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=2=count(//lst[@name='segments']/lst/str[@name='version'][.='7.0.0'])
[junit4] > xml response was: <?xml version="1.0" encoding="UTF-8"?>
[junit4] > <response>
[junit4] > <lst name="responseHeader"><int name="status">0</int><int name="QTime">3</int></lst><lst name="segments"><lst name="_0"><str name="name">_0</str><int name="delCount">1</int><long name="sizeInBytes">1732</long><int name="size">4</int><date name="age">2017-03-05T17:35:43.518Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst><lst name="_1"><str name="name">_1</str><int name="delCount">0</int><long name="sizeInBytes">1581</long><int name="size">1</int><date name="age">2017-03-05T17:35:43.520Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst><lst name="_2"><str name="name">_2</str><int name="delCount">0</int><long name="sizeInBytes">1665</long><int name="size">4</int><date name="age">2017-03-05T17:35:43.527Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst><lst name="_3"><str name="name">_3</str><int name="delCount">0</int><long name="sizeInBytes">1581</long><int name="size">1</int><date name="age">2017-03-05T17:35:43.528Z</date><str name="source">flush</str><str name="version">7.0.0</str></lst></lst>
[junit4] > </response>
[junit4] > request was:qt=/admin/segments&wt=xml
[junit4] > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:912)
[junit4] > ... 40 more
[junit4] 2> 3718934 INFO (TEST-SegmentsInfoRequestHandlerTest.testSegmentInfos-seed#[A5A0B1F6AC0C365C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSegmentInfos
[junit4] 2> 3718934 INFO (TEST-SegmentsInfoRequestHandlerTest.testSegmentInfos-seed#[A5A0B1F6AC0C365C]) [ ] o.a.s.c.S.Request [collection1] webapp=null path=null params={qt=/admin/segments&wt=xml} status=0 QTime=0
[junit4] 2> 3718935 INFO (TEST-SegmentsInfoRequestHandlerTest.testSegmentInfos-seed#[A5A0B1F6AC0C365C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSegmentInfos
[junit4] 2> 3718935 INFO (SUITE-SegmentsInfoRequestHandlerTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> 3718935 INFO (SUITE-SegmentsInfoRequestHandlerTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1549558850
[junit4] 2> 3718935 INFO (coreCloseExecutor-7622-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@493e70cf
[junit4] 2> 3718957 INFO (coreCloseExecutor-7622-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 3718959 INFO (SUITE-SegmentsInfoRequestHandlerTest-seed#[A5A0B1F6AC0C365C]-worker) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.handler.admin.SegmentsInfoRequestHandlerTest_A5A0B1F6AC0C365C-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {name=FSTOrd50, id=FSTOrd50}, docValues:{}, maxPointsInLeafNode=710, maxMBSortInHeap=6.510209896215786, sim=RandomSimilarity(queryNorm=true): {}, locale=zh, timezone=Brazil/Acre
[junit4] 2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=4,threads=1,free=208588576,total=529530880
[junit4] 2> NOTE: All tests run in this JVM: [DirectSolrSpellCheckerTest, TestClassNameShortening, CleanupOldIndexTest, TestSchemaNameResource, HdfsCollectionsAPIDistributedZkTest, CollectionStateFormat2Test, TestDeleteCollectionOnDownNodes, TriLevelCompositeIdRoutingTest, CdcrRequestHandlerTest, TestExclusionRuleCollectionAccess, UpdateRequestProcessorFactoryTest, MoreLikeThisHandlerTest, UUIDUpdateProcessorFallbackTest, TestManagedSynonymFilterFactory, TestDocBasedVersionConstraints, SampleTest, AddBlockUpdateTest, HdfsLockFactoryTest, TestNRTOpen, TestRandomRequestDistribution, TestObjectReleaseTracker, TestSolrCloudWithDelegationTokens, CSVRequestHandlerTest, DistributedFacetPivotSmallTest, CheckHdfsIndexTest, InfoHandlerTest, TestStressUserVersions, TestRandomCollapseQParserPlugin, EnumFieldTest, BlockJoinFacetSimpleTest, TestUpdate, ShardRoutingTest, OverseerStatusTest, TestPartialUpdateDeduplication, TestScoreJoinQPScore, TestSurroundQueryParser, TestRecoveryHdfs, TestXIncludeConfig, SpellCheckCollatorTest, HdfsWriteToMultipleCollectionsTest, SpatialFilterTest, ChaosMonkeySafeLeaderTest, SchemaVersionSpecificBehaviorTest, TestSimpleQParserPlugin, TestAnalyzeInfixSuggestions, ZkCLITest, DistributedDebugComponentTest, IndexSchemaTest, BJQParserTest, ZkSolrClientTest, TermsComponentTest, TestFuzzyAnalyzedSuggestions, DistributedMLTComponentTest, TestQueryUtils, TestSystemIdResolver, TestSQLHandler, VersionInfoTest, CoreAdminCreateDiscoverTest, TestManagedSchema, TestReplicationHandlerBackup, TestManagedResourceStorage, HdfsBasicDistributedZkTest, DocValuesTest, TestBinaryResponseWriter, SolrCmdDistributorTest, TestSolrQueryResponse, TestFreeTextSuggestions, InfixSuggestersTest, SpellingQueryConverterTest, SolrInfoMBeanTest, TestUnifiedSolrHighlighter, DistributedFacetPivotLargeTest, TestConfigSetImmutable, TestFieldResource, SolrIndexSplitterTest, ConnectionManagerTest, TestSweetSpotSimilarityFactory, TestPointFields, TestClusterProperties, IndexSchemaRuntimeFieldTest, TestRequestForwarding, TestEmbeddedSolrServerConstructors, DistanceFunctionTest, DistributedQueryComponentOptimizationTest, SimplePostToolTest, TestSolrConfigHandler, AlternateDirectoryTest, TestMiniSolrCloudClusterSSL, TestDefaultStatsCache, TestCursorMarkWithoutUniqueKey, SharedFSAutoReplicaFailoverTest, ZkStateReaderTest, QueryParsingTest, TestNumericTerms32, DistributedQueueTest, TestCloudPivotFacet, TestHashPartitioner, TestWordDelimiterFilterFactory, LeaderInitiatedRecoveryOnShardRestartTest, DOMUtilTest, TestBlobHandler, TestStressCloudBlindAtomicUpdates, SolrJmxReporterTest, BasicFunctionalityTest, DistributedTermsComponentTest, TestPostingsSolrHighlighter, ReplicationFactorTest, TestDocumentBuilder, DistributedIntervalFacetingTest, SolrCLIZkUtilsTest, TestLegacyNumericRangeQueryBuilder, HighlighterTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerTaskQueueTest, RemoteQueryErrorTest, RestartWhileUpdatingTest, SharedFSAutoReplicaFailoverUtilsTest, SolrCloudExampleTest, TestAuthenticationFramework, TestCloudDeleteByQuery, TestCloudInspectUtil, TestConfigSetsAPIExclusivity, TestCryptoKeys, TestDistribDocBasedVersion, TestLeaderElectionZkExpiry, TestLockTree, TestMiniSolrCloudCluster, TestRandomFlRTGCloud, TestSolrCloudWithSecureImpersonation, HdfsNNFailoverTest, HdfsThreadLeakTest, StressHdfsTest, RuleEngineTest, BlobRepositoryCloudTest, TestDynamicLoading, RequestLoggingTest, TestConfigReload, SegmentsInfoRequestHandlerTest]
[junit4] Completed [561/693 (2!)] on J1 in 0.38s, 3 tests, 1 error <<< FAILURES!
[...truncated 63018 lines...]