You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2016/12/15 08:36:00 UTC
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 555 -
Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/555/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
2 tests failed.
FAILED: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail
Error Message:
expected:<200> but was:<404>
Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
at __randomizedtesting.SeedInfo.seed([AF8E0FECB28DF132:C7313AC66217E3DE]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:141)
at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:295)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130) at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102) at sun.reflect.GeneratedConstructorAccessor190.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
at sun.reflect.GeneratedConstructorAccessor190.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)
at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)
at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([AF8E0FECB28DF132]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
at sun.reflect.GeneratedMethodAccessor21.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 11243 lines...]
[junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_AF8E0FECB28DF132-001/init-core-data-001
[junit4] 2> 783062 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
[junit4] 2> 783066 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_AF8E0FECB28DF132-001/tempDir-001
[junit4] 2> 783066 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 783066 INFO (Thread-6465) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 783066 INFO (Thread-6465) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 783167 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:40534
[junit4] 2> 783177 INFO (jetty-launcher-1271-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 783177 INFO (jetty-launcher-1271-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 783179 INFO (jetty-launcher-1271-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@54a9b394{/solr,null,AVAILABLE}
[junit4] 2> 783180 INFO (jetty-launcher-1271-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@39aaa939{HTTP/1.1,[http/1.1]}{127.0.0.1:64360}
[junit4] 2> 783180 INFO (jetty-launcher-1271-thread-1) [ ] o.e.j.s.Server Started @786288ms
[junit4] 2> 783180 INFO (jetty-launcher-1271-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=64360}
[junit4] 2> 783181 ERROR (jetty-launcher-1271-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 783181 INFO (jetty-launcher-1271-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@38dca9c2{/solr,null,AVAILABLE}
[junit4] 2> 783181 INFO (jetty-launcher-1271-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 783181 INFO (jetty-launcher-1271-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 783181 INFO (jetty-launcher-1271-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 783181 INFO (jetty-launcher-1271-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-15T07:41:48.655Z
[junit4] 2> 783181 INFO (jetty-launcher-1271-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2fffb40c{HTTP/1.1,[http/1.1]}{127.0.0.1:47838}
[junit4] 2> 783181 INFO (jetty-launcher-1271-thread-2) [ ] o.e.j.s.Server Started @786289ms
[junit4] 2> 783181 INFO (jetty-launcher-1271-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=47838}
[junit4] 2> 783181 ERROR (jetty-launcher-1271-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 783182 INFO (jetty-launcher-1271-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 783182 INFO (jetty-launcher-1271-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 783182 INFO (jetty-launcher-1271-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 783182 INFO (jetty-launcher-1271-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-15T07:41:48.656Z
[junit4] 2> 783185 INFO (jetty-launcher-1271-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 783185 INFO (jetty-launcher-1271-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 783194 INFO (jetty-launcher-1271-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 783194 INFO (jetty-launcher-1271-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 783195 INFO (jetty-launcher-1271-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40534/solr
[junit4] 2> 783196 INFO (jetty-launcher-1271-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40534/solr
[junit4] 2> 783238 INFO (jetty-launcher-1271-thread-1) [n:127.0.0.1:64360_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:64360_solr
[junit4] 2> 783239 INFO (jetty-launcher-1271-thread-2) [n:127.0.0.1:47838_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47838_solr
[junit4] 2> 783239 INFO (jetty-launcher-1271-thread-1) [n:127.0.0.1:64360_solr ] o.a.s.c.Overseer Overseer (id=97110439267139590-127.0.0.1:64360_solr-n_0000000000) starting
[junit4] 2> 783242 INFO (zkCallback-1282-thread-1-processing-n:127.0.0.1:64360_solr) [n:127.0.0.1:64360_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 783253 INFO (jetty-launcher-1271-thread-1) [n:127.0.0.1:64360_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64360_solr
[junit4] 2> 783254 INFO (zkCallback-1281-thread-1-processing-n:127.0.0.1:47838_solr) [n:127.0.0.1:47838_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 783256 INFO (zkCallback-1281-thread-1-processing-n:127.0.0.1:47838_solr) [n:127.0.0.1:47838_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 783258 INFO (OverseerStateUpdate-97110439267139590-127.0.0.1:64360_solr-n_0000000000) [n:127.0.0.1:64360_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 783333 INFO (jetty-launcher-1271-thread-2) [n:127.0.0.1:47838_solr ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 783346 INFO (jetty-launcher-1271-thread-2) [n:127.0.0.1:47838_solr ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
[junit4] 2> 783451 INFO (jetty-launcher-1271-thread-2) [n:127.0.0.1:47838_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 783451 INFO (jetty-launcher-1271-thread-1) [n:127.0.0.1:64360_solr ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 783451 INFO (jetty-launcher-1271-thread-1) [n:127.0.0.1:64360_solr ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
[junit4] 2> 783452 INFO (jetty-launcher-1271-thread-1) [n:127.0.0.1:64360_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 783485 INFO (jetty-launcher-1271-thread-1-EventThread) [n:127.0.0.1:64360_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 783487 INFO (jetty-launcher-1271-thread-2-EventThread) [n:127.0.0.1:47838_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 783635 INFO (jetty-launcher-1271-thread-2) [n:127.0.0.1:47838_solr ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@1ac9276e
[junit4] 2> 783635 INFO (jetty-launcher-1271-thread-2) [n:127.0.0.1:47838_solr ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@1ac9276e
[junit4] 2> 783636 INFO (jetty-launcher-1271-thread-1) [n:127.0.0.1:64360_solr ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@2313bd49
[junit4] 2> 783636 INFO (jetty-launcher-1271-thread-2) [n:127.0.0.1:47838_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_AF8E0FECB28DF132-001/tempDir-001/node2/.
[junit4] 2> 783636 INFO (jetty-launcher-1271-thread-1) [n:127.0.0.1:64360_solr ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@2313bd49
[junit4] 2> 783639 INFO (jetty-launcher-1271-thread-1) [n:127.0.0.1:64360_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_AF8E0FECB28DF132-001/tempDir-001/node1/.
[junit4] 2> 783686 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
[junit4] 2> 783756 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 783758 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 783773 INFO (qtp370379419-12138) [n:127.0.0.1:47838_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhb9igFZJXqa_QEDFMILYURSoSY7Azqs-eJhu4dDi13qB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 783773 INFO (qtp370379419-12138) [n:127.0.0.1:47838_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhb9igFZJXqa_QEDFMILYURSoSY7Azqs-eJhu4dDi13qB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 783779 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 783780 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 783787 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 783793 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 783794 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 783796 INFO (qtp370379419-12139) [n:127.0.0.1:47838_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhb9igFZJXqa_QEDFMILYURSoSY7Azqs-eJhu4dDi13qB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 783796 INFO (qtp370379419-12139) [n:127.0.0.1:47838_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhb9igFZJXqa_QEDFMILYURSoSY7Azqs-eJhu4dDi13qB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 783801 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 783802 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
[junit4] 2> 783828 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
[junit4] 2> 783845 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 783847 INFO (qtp370379419-12140) [n:127.0.0.1:47838_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhd7igFZJXqbewIDFC9ayVxYD7qdXyGz61-G9llfM7urB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 783847 INFO (qtp370379419-12140) [n:127.0.0.1:47838_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhd7igFZJXqbewIDFC9ayVxYD7qdXyGz61-G9llfM7urB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 783848 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
[junit4] 2> 783881 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
[junit4] 2> 783907 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSolrCloudWithDelegationTokens -Dtests.method=testDelegationTokenCancelFail -Dtests.seed=AF8E0FECB28DF132 -Dtests.slow=true -Dtests.locale=cs -Dtests.timezone=Asia/Ujung_Pandang -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 0.06s J1 | TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail <<<
[junit4] > Throwable #1: java.lang.AssertionError: expected:<200> but was:<404>
[junit4] > at __randomizedtesting.SeedInfo.seed([AF8E0FECB28DF132:C7313AC66217E3DE]:0)
[junit4] > at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:141)
[junit4] > at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:295)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 783939 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
[junit4] 2> 783949 WARN (qtp370379419-12136) [ ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
[junit4] 2> 783949 WARN (qtp370379419-12136) [ ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 783949 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 783952 WARN (qtp1826947984-12130) [ ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
[junit4] 2> 783952 WARN (qtp1826947984-12130) [ ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 783952 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 783961 WARN (qtp370379419-12131) [ ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
[junit4] 2> 783961 WARN (qtp370379419-12131) [ ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 783961 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 783963 WARN (qtp1826947984-12123) [ ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
[junit4] 2> 783963 WARN (qtp1826947984-12123) [ ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 783964 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 783964 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
[junit4] 2> 783993 INFO (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
[junit4] 2> 784002 INFO (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
[junit4] 2> 784038 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
[junit4] 2> 784055 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 784057 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 784059 INFO (qtp370379419-12140) [n:127.0.0.1:47838_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhhLigFZJXqcSwcDFOW8w-wuFbVC41BhIQRb4szFeD30B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 784059 INFO (qtp370379419-12140) [n:127.0.0.1:47838_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhhLigFZJXqcSwcDFOW8w-wuFbVC41BhIQRb4szFeD30B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 784065 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 784067 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 784069 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 784075 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 784077 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 784080 INFO (qtp370379419-12140) [n:127.0.0.1:47838_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhhLigFZJXqcSwcDFOW8w-wuFbVC41BhIQRb4szFeD30B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 784080 INFO (qtp370379419-12140) [n:127.0.0.1:47838_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhhLigFZJXqcSwcDFOW8w-wuFbVC41BhIQRb4szFeD30B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 784085 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 784086 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 784089 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 784108 INFO (qtp370379419-12140) [n:127.0.0.1:47838_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhh-igFZJXqcfggDFJyndTWO5spVmK2Nqhq2XIQnPgJ2B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 784108 INFO (qtp370379419-12140) [n:127.0.0.1:47838_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhh-igFZJXqcfggDFJyndTWO5spVmK2Nqhq2XIQnPgJ2B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 784110 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 784113 INFO (qtp1826947984-12128) [n:127.0.0.1:64360_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhh-igFZJXqcfggDFJyndTWO5spVmK2Nqhq2XIQnPgJ2B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 784113 INFO (qtp1826947984-12128) [n:127.0.0.1:64360_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhh-igFZJXqcfggDFJyndTWO5spVmK2Nqhq2XIQnPgJ2B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 784117 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 784119 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 784121 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 784135 INFO (qtp370379419-12139) [n:127.0.0.1:47838_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhieigFZJXqcngkDFPLBvbuukI42hhB-dqZ3PskWFKj1B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 784136 INFO (qtp370379419-12139) [n:127.0.0.1:47838_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhieigFZJXqcngkDFPLBvbuukI42hhB-dqZ3PskWFKj1B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 784138 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 784142 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 784143 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 784145 INFO (qtp370379419-12139) [n:127.0.0.1:47838_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhieigFZJXqcngkDFPLBvbuukI42hhB-dqZ3PskWFKj1B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 784145 INFO (qtp370379419-12139) [n:127.0.0.1:47838_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhieigFZJXqcngkDFPLBvbuukI42hhB-dqZ3PskWFKj1B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 784149 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 784163 INFO (qtp370379419-12138) [n:127.0.0.1:47838_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhi5igFZJXqcuQoDFH9Frvj_xagNZ-h4GxyEyU5uojI4B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 784163 INFO (qtp370379419-12138) [n:127.0.0.1:47838_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhi5igFZJXqcuQoDFH9Frvj_xagNZ-h4GxyEyU5uojI4B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 784168 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 784169 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 784172 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 784177 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 784178 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 784180 INFO (qtp370379419-12137) [n:127.0.0.1:47838_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhi5igFZJXqcuQoDFH9Frvj_xagNZ-h4GxyEyU5uojI4B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 784180 INFO (qtp370379419-12137) [n:127.0.0.1:47838_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkBbhi5igFZJXqcuQoDFH9Frvj_xagNZ-h4GxyEyU5uojI4B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 784184 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 784184 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
[junit4] 2> 784215 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
[junit4] 2> 784228 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 784229 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 784232 ERROR (qtp370379419-12136) [ ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_11]
[junit4] 2> 784244 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 784245 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 784247 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 784250 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 784265 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 784267 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 784269 ERROR (qtp370379419-12137) [n:127.0.0.1:47838_solr ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_12]
[junit4] 2> 784276 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 784278 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 784280 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 784284 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= t, 01-Led-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: t, 01-Led-1970 00:00:00 GMT
[junit4] 2> 784284 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
[junit4] 2> 784310 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
[junit4] 2> 784314 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 784316 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[AF8E0FECB28DF132]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40534/solr ready
[junit4] 2> 784318 INFO (qtp370379419-12131) [n:127.0.0.1:47838_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 784318 INFO (qtp370379419-12131) [n:127.0.0.1:47838_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 784319 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[AF8E0FECB28DF132]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=test&e=1481823709792&s=1mUU6CTn2bDJNuYcZPELvROz9YI="; Path=/; Domain=127.0.0.1; Expires= t, 15-Pro-2016 17:41:49 GMT; HttpOnly". Invalid 'expires' attribute: t, 15-Pro-2016 17:41:49 GMT
[junit4] 2> 784320 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
[junit4] 2> 784321 INFO (jetty-launcher-1271-thread-2) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@39aaa939{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 784322 INFO (jetty-launcher-1271-thread-2) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1371183145
[junit4] 2> 784323 INFO (jetty-launcher-1271-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@2fffb40c{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 784323 INFO (jetty-launcher-1271-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1352799876
[junit4] 2> 784333 INFO (zkCallback-1282-thread-3-processing-n:127.0.0.1:64360_solr) [n:127.0.0.1:64360_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 784334 WARN (zkCallback-1281-thread-1-processing-n:127.0.0.1:47838_solr) [n:127.0.0.1:47838_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 784334 INFO (jetty-launcher-1271-thread-2) [ ] o.a.s.c.Overseer Overseer (id=97110439267139590-127.0.0.1:64360_solr-n_0000000000) closing
[junit4] 2> 784335 INFO (OverseerStateUpdate-97110439267139590-127.0.0.1:64360_solr-n_0000000000) [n:127.0.0.1:64360_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:64360_solr
[junit4] 2> 784333 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x159016e14800005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 784337 ERROR (Thread[Thread-6482,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:47838_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 784339 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x159016e14800006, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 784340 ERROR (Thread[Thread-6483,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:64360_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 784352 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x159016e14800007, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 784352 INFO (jetty-launcher-1271-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@38dca9c2{/solr,null,UNAVAILABLE}
[junit4] 2> 784352 INFO (jetty-launcher-1271-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@54a9b394{/solr,null,UNAVAILABLE}
[junit4] 2> 784356 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40534 40534
[junit4] 2> 784370 INFO (Thread-6465) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40534 40534
[junit4] 2> 784372 WARN (Thread-6465) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 18 /solr/aliases.json
[junit4] 2> 18 /solr/clusterprops.json
[junit4] 2> 2 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 18 /solr/clusterstate.json
[junit4] 2> 4 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_9
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_12
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_11
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 18 /solr/live_nodes
[junit4] 2> 18 /solr/collections
[junit4] 2> 3 /solr/overseer/queue
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
[junit4] 2>
[junit4] 2> 784373 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_AF8E0FECB28DF132-001
[junit4] 2> Dec 15, 2016 7:41:49 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 7 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true,coord=crazy): {}, locale=cs, timezone=Asia/Ujung_Pandang
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=1,free=147571624,total=461373440
[junit4] 2> NOTE: All tests run in this JVM: [FieldMutatingUpdateProcessorTest, TestQueryTypes, ParsingFieldUpdateProcessorsTest, TestIndexingPerformance, TestSSLRandomization, TestNRTOpen, SolrCloudExampleTest, RequestLoggingTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestCustomSort, TestDFISimilarityFactory, NotRequiredUniqueKeyTest, AliasIntegrationTest, OverriddenZkACLAndCredentialsProvidersTest, TestWriterPerf, TestFilteredDocIdSet, DirectUpdateHandlerOptimizeTest, TestAuthenticationFramework, URLClassifyProcessorTest, TestConfigOverlay, CacheHeaderTest, LoggingHandlerTest, TestSortByMinMaxFunction, TestClusterProperties, TestHashQParserPlugin, TestCoreDiscovery, TestConfigSetProperties, TestQuerySenderListener, TestQuerySenderNoQuery, DistribCursorPagingTest, TestSolrCloudSnapshots, CheckHdfsIndexTest, ChaosMonkeySafeLeaderTest, TestRangeQuery, ExternalFileFieldSortTest, CurrencyFieldOpenExchangeTest, TestSuggestSpellingConverter, EchoParamsTest, ClusterStateTest, TestCorePropertiesReload, BlockCacheTest, TestShardHandlerFactory, TestFuzzyAnalyzedSuggestions, TestRTimerTree, CursorMarkTest, TestInfoStreamLogging, SuggesterTest, TestSimpleTrackingShardHandler, TestConfigSetsAPIZkFailure, DataDrivenBlockJoinTest, TestRandomFlRTGCloud, ShardRoutingCustomTest, TestMaxScoreQueryParser, DistribJoinFromCollectionTest, TestAtomicUpdateErrorCases, TestPerFieldSimilarityClassic, TestHdfsUpdateLog, TestDocSet, TestRestoreCore, TestCloudInspectUtil, DeleteInactiveReplicaTest, BasicDistributedZk2Test, TestEmbeddedSolrServerConstructors, CdcrReplicationHandlerTest, TestNumericTerms32, TestCloudSchemaless, TestConfigSetsAPI, DeleteNodeTest, FieldAnalysisRequestHandlerTest, TestRequestForwarding, CoreSorterTest, TestSchemaSimilarityResource, ForceLeaderTest, NoCacheHeaderTest, TestFieldCacheWithThreads, TestSearcherReuse, TestStressUserVersions, ChangedSchemaMergeTest, TestCrossCoreJoin, TestPHPSerializedResponseWriter, ConnectionReuseTest, TestDeleteCollectionOnDownNodes, TestLMDirichletSimilarityFactory, TestConfigReload, SearchHandlerTest, TestSerializedLuceneMatchVersion, HdfsRestartWhileUpdatingTest, TestDFRSimilarityFactory, TestLMJelinekMercerSimilarityFactory, SpatialHeatmapFacetsTest, DistanceFunctionTest, TestManagedResourceStorage, TestDownShardTolerantSearch, TestSlowCompositeReaderWrapper, DistributedIntervalFacetingTest, MBeansHandlerTest, TestLuceneMatchVersion, TestSchemaNameResource, SuggesterWFSTTest, TestSolrCloudWithDelegationTokens]
[junit4] Completed [192/662 (1!)] on J1 in 1.59s, 8 tests, 1 failure <<< FAILURES!
[...truncated 1124 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_AF8E0FECB28DF132-001/init-core-data-001
[junit4] 2> 2447503 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 2447504 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_AF8E0FECB28DF132-001/tempDir-001
[junit4] 2> 2447505 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2447505 INFO (Thread-5040) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2447505 INFO (Thread-5040) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2447606 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:34586
[junit4] 2> 2447617 INFO (jetty-launcher-9412-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2447618 INFO (jetty-launcher-9412-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2447621 INFO (jetty-launcher-9412-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6ef0371f{/solr,null,AVAILABLE}
[junit4] 2> 2447621 INFO (jetty-launcher-9412-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@16c89073{/solr,null,AVAILABLE}
[junit4] 2> 2447621 INFO (jetty-launcher-9412-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@39c2e9dd{HTTP/1.1,[http/1.1]}{127.0.0.1:47284}
[junit4] 2> 2447621 INFO (jetty-launcher-9412-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1f12ada8{HTTP/1.1,[http/1.1]}{127.0.0.1:58481}
[junit4] 2> 2447621 INFO (jetty-launcher-9412-thread-2) [ ] o.e.j.s.Server Started @2451502ms
[junit4] 2> 2447621 INFO (jetty-launcher-9412-thread-1) [ ] o.e.j.s.Server Started @2451502ms
[junit4] 2> 2447621 INFO (jetty-launcher-9412-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=47284}
[junit4] 2> 2447621 INFO (jetty-launcher-9412-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58481}
[junit4] 2> 2447622 ERROR (jetty-launcher-9412-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2447622 INFO (jetty-launcher-9412-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 2447623 INFO (jetty-launcher-9412-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2447623 INFO (jetty-launcher-9412-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2447623 INFO (jetty-launcher-9412-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-15T08:09:32.991Z
[junit4] 2> 2447622 ERROR (jetty-launcher-9412-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2447623 INFO (jetty-launcher-9412-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 2447623 INFO (jetty-launcher-9412-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2447624 INFO (jetty-launcher-9412-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2447624 INFO (jetty-launcher-9412-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-15T08:09:32.992Z
[junit4] 2> 2447627 INFO (jetty-launcher-9412-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2447628 INFO (jetty-launcher-9412-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2447634 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x159018779ca0002, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2447634 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x159018779ca0001, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2447635 INFO (jetty-launcher-9412-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2447637 INFO (jetty-launcher-9412-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2447637 INFO (jetty-launcher-9412-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34586/solr
[junit4] 2> 2447638 INFO (jetty-launcher-9412-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34586/solr
[junit4] 2> 2447677 INFO (jetty-launcher-9412-thread-2) [n:127.0.0.1:47284_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:47284_solr
[junit4] 2> 2447678 INFO (jetty-launcher-9412-thread-1) [n:127.0.0.1:58481_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58481_solr
[junit4] 2> 2447680 INFO (jetty-launcher-9412-thread-2) [n:127.0.0.1:47284_solr ] o.a.s.c.Overseer Overseer (id=97110548340670470-127.0.0.1:47284_solr-n_0000000000) starting
[junit4] 2> 2447680 INFO (zkCallback-9423-thread-1-processing-n:127.0.0.1:47284_solr) [n:127.0.0.1:47284_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2447680 INFO (zkCallback-9422-thread-1-processing-n:127.0.0.1:58481_solr) [n:127.0.0.1:58481_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2447694 INFO (jetty-launcher-9412-thread-2) [n:127.0.0.1:47284_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47284_solr
[junit4] 2> 2447698 INFO (zkCallback-9423-thread-1-processing-n:127.0.0.1:47284_solr) [n:127.0.0.1:47284_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2447698 INFO (zkCallback-9422-thread-1-processing-n:127.0.0.1:58481_solr) [n:127.0.0.1:58481_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2447757 INFO (jetty-launcher-9412-thread-1) [n:127.0.0.1:58481_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_AF8E0FECB28DF132-001/tempDir-001/node1/.
[junit4] 2> 2447824 INFO (jetty-launcher-9412-thread-2) [n:127.0.0.1:47284_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_AF8E0FECB28DF132-001/tempDir-001/node2/.
[junit4] 2> 2447832 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2447834 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34586/solr ready
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 2447889 WARN (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 2447898 WARN (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2447901 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2447918 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_solaris.vm_44299_hdfs____.n1s6cc/webapp
[junit4] 2> 2448424 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:44299
[junit4] 2> 2448528 WARN (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2448531 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2448545 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_60038_datanode____.5zzkzi/webapp
[junit4] 2> 2449091 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:60038
[junit4] 2> 2449158 WARN (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2449159 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2449195 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_44324_datanode____i14qzg/webapp
[junit4] 2> 2449252 INFO (IPC Server handler 2 on 33650) [ ] BlockStateChange BLOCK* processReport: from storage DS-e4280b5b-7b03-4134-a610-1ac3909cacb2 node DatanodeRegistration(127.0.0.1:55916, datanodeUuid=82e6b4f1-e8ac-4c8f-b7f2-4456f16f71f4, infoPort=53218, infoSecurePort=0, ipcPort=53220, storageInfo=lv=-56;cid=testClusterID;nsid=1144978801;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2449252 INFO (IPC Server handler 2 on 33650) [ ] BlockStateChange BLOCK* processReport: from storage DS-925d2046-6586-4bc0-9abc-86b129eac940 node DatanodeRegistration(127.0.0.1:55916, datanodeUuid=82e6b4f1-e8ac-4c8f-b7f2-4456f16f71f4, infoPort=53218, infoSecurePort=0, ipcPort=53220, storageInfo=lv=-56;cid=testClusterID;nsid=1144978801;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2449717 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44324
[junit4] 2> 2449813 INFO (IPC Server handler 5 on 33650) [ ] BlockStateChange BLOCK* processReport: from storage DS-8c0e8556-78d0-409e-87aa-71cf4e674f6b node DatanodeRegistration(127.0.0.1:48936, datanodeUuid=318d11e6-63ff-4c89-ab23-4c254071866c, infoPort=54387, infoSecurePort=0, ipcPort=65472, storageInfo=lv=-56;cid=testClusterID;nsid=1144978801;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2449813 INFO (IPC Server handler 5 on 33650) [ ] BlockStateChange BLOCK* processReport: from storage DS-8e68e8e4-d0c1-4e29-babb-594f059efded node DatanodeRegistration(127.0.0.1:48936, datanodeUuid=318d11e6-63ff-4c89-ab23-4c254071866c, infoPort=54387, infoSecurePort=0, ipcPort=65472, storageInfo=lv=-56;cid=testClusterID;nsid=1144978801;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2449917 INFO (TEST-HdfsRecoveryZkTest.test-seed#[AF8E0FECB28DF132]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2449919 INFO (qtp1031681066-42365) [n:127.0.0.1:58481_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2449923 INFO (OverseerThreadFactory-10597-thread-1-processing-n:127.0.0.1:47284_solr) [n:127.0.0.1:47284_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 2450032 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 2450032 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 2450141 INFO (zkCallback-9423-thread-1-processing-n:127.0.0.1:47284_solr) [n:127.0.0.1:47284_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2450141 INFO (zkCallback-9422-thread-1-processing-n:127.0.0.1:58481_solr) [n:127.0.0.1:58481_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2451046 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 2451046 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 2451073 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema [recoverytest_shard1_replica1] Schema name=minimal
[junit4] 2> 2451073 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema [recoverytest_shard1_replica2] Schema name=minimal
[junit4] 2> 2451076 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2451076 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2451078 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica2' using configuration from collection recoverytest
[junit4] 2> 2451078 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica1' using configuration from collection recoverytest
[junit4] 2> 2451085 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:33650/data
[junit4] 2> 2451085 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 2451086 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [[recoverytest_shard1_replica2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_AF8E0FECB28DF132-001/tempDir-001/node1/recoverytest_shard1_replica2], dataDir=[hdfs://solaris-vm:33650/data/recoverytest/core_node1/data/]
[junit4] 2> 2451086 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:33650/data
[junit4] 2> 2451086 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:33650/data/recoverytest/core_node1/data/snapshot_metadata
[junit4] 2> 2451086 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 2451086 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [[recoverytest_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_AF8E0FECB28DF132-001/tempDir-001/node2/recoverytest_shard1_replica1], dataDir=[hdfs://solaris-vm:33650/data/recoverytest/core_node2/data/]
[junit4] 2> 2451086 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:33650/data/recoverytest/core_node2/data/snapshot_metadata
[junit4] 2> 2451095 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2451095 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2451095 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2451096 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2451095 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2451096 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2451160 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2451161 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:33650/data/recoverytest/core_node2/data
[junit4] 2> 2451177 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2451179 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:33650/data/recoverytest/core_node1/data
[junit4] 2> 2451190 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:33650/data/recoverytest/core_node2/data/index
[junit4] 2> 2451198 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2451198 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2451198 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2451199 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:33650/data/recoverytest/core_node1/data/index
[junit4] 2> 2451206 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2451206 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2451206 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2451281 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2451284 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2451602 INFO (IPC Server handler 5 on 33650) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:48936 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-925d2046-6586-4bc0-9abc-86b129eac940:NORMAL:127.0.0.1:55916|RBW], ReplicaUC[[DISK]DS-8e68e8e4-d0c1-4e29-babb-594f059efded:NORMAL:127.0.0.1:48936|FINALIZED]]} size 0
[junit4] 2> 2451603 INFO (IPC Server handler 4 on 33650) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:55916 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-8e68e8e4-d0c1-4e29-babb-594f059efded:NORMAL:127.0.0.1:48936|FINALIZED], ReplicaUC[[DISK]DS-e4280b5b-7b03-4134-a610-1ac3909cacb2:NORMAL:127.0.0.1:55916|FINALIZED]]} size 0
[junit4] 2> 2451604 INFO (IPC Server handler 8 on 33650) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:48936 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-925d2046-6586-4bc0-9abc-86b129eac940:NORMAL:127.0.0.1:55916|RBW], ReplicaUC[[DISK]DS-8c0e8556-78d0-409e-87aa-71cf4e674f6b:NORMAL:127.0.0.1:48936|RBW]]} size 0
[junit4] 2> 2451604 INFO (IPC Server handler 6 on 33650) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:55916 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-925d2046-6586-4bc0-9abc-86b129eac940:NORMAL:127.0.0.1:55916|RBW], ReplicaUC[[DISK]DS-8c0e8556-78d0-409e-87aa-71cf4e674f6b:NORMAL:127.0.0.1:48936|RBW]]} size 0
[junit4] 2> 2451620 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 2451620 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2451620 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 2451630 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 2451630 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2451630 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 2451632 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2451632 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2451646 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2451646 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2451656 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@7303482e[recoverytest_shard1_replica2] main]
[junit4] 2> 2451658 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2451659 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2451659 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2451660 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1553768777806512128
[junit4] 2> 2451660 INFO (searcherExecutor-10602-thread-1-processing-n:127.0.0.1:58481_solr x:recoverytest_shard1_replica2 s:shard1 c:recoverytest) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [recoverytest_shard1_replica2] Registered new searcher Searcher@7303482e[recoverytest_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2451667 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@282f2b03[recoverytest_shard1_replica1] main]
[junit4] 2> 2451669 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2451670 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2451670 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2451670 INFO (searcherExecutor-10603-thread-1-processing-n:127.0.0.1:47284_solr x:recoverytest_shard1_replica1 s:shard1 c:recoverytest) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [recoverytest_shard1_replica1] Registered new searcher Searcher@282f2b03[recoverytest_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2451670 INFO (qtp103462449-42357) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1553768777816997888
[junit4] 2> 2451671 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 2451673 INFO (zkCallback-9423-thread-2-processing-n:127.0.0.1:47284_solr) [n:127.0.0.1:47284_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2451674 INFO (zkCallback-9422-thread-1-processing-n:127.0.0.1:58481_solr) [n:127.0.0.1:58481_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2452180 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2452180 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2452180 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:58481/solr/recoverytest_shard1_replica2/
[junit4] 2> 2452181 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:58481/solr START replicas=[http://127.0.0.1:47284/solr/recoverytest_shard1_replica1/] nUpdates=100
[junit4] 2> 2452181 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:58481/solr DONE. We have no versions. sync failed.
[junit4] 2> 2452184 INFO (qtp103462449-42360) [n:127.0.0.1:47284_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.S.Request [recoverytest_shard1_replica1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2452184 INFO (qtp1031681066-42367) [n:127.0.0.1:58481_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[juni
[...truncated too long message...]
getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> 2465292 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34586 34586
[junit4] 2> 2465293 INFO (SUITE-HdfsRecoveryZkTest-seed#[AF8E0FECB28DF132]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {rnd_b=PostingsFormat(name=LuceneFixedGap), _version_=PostingsFormat(name=Memory doPackFST= false), a_t=PostingsFormat(name=Memory doPackFST= false), a_i=PostingsFormat(name=LuceneFixedGap), id=PostingsFormat(name=LuceneFixedGap)}, docValues:{}, maxPointsInLeafNode=310, maxMBSortInHeap=6.384885969277208, sim=RandomSimilarity(queryNorm=false,coord=yes): {}, locale=ar-EG, timezone=Africa/Bujumbura
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=2,free=115312024,total=492830720
[junit4] 2> NOTE: All tests run in this JVM: [TestClusterStateMutator, TestFieldCacheSort, TestPKIAuthenticationPlugin, TestRemoteStreaming, SimplePostToolTest, TestReRankQParserPlugin, TestExactSharedStatsCache, SpatialFilterTest, TestImplicitCoreProperties, TestCloudDeleteByQuery, TestUniqueKeyFieldResource, PreAnalyzedUpdateProcessorTest, TestFileDictionaryLookup, ClusterStateUpdateTest, TestPhraseSuggestions, SimpleFacetsTest, AnalyticsQueryTest, TestLazyCores, TestPivotHelperCode, TestSolrCLIRunExample, CursorPagingTest, DistributedSpellCheckComponentTest, TestSolr4Spatial2, TestManagedSchemaAPI, TestCSVResponseWriter, OverseerRolesTest, HdfsTlogReplayBufferedWhileIndexingTest, DistributedFacetPivotLargeTest, TestUseDocValuesAsStored2, TestFastLRUCache, TestConfigSets, MultiTermTest, TestLegacyFieldCache, TestStressRecovery, SecurityConfHandlerTest, TestLRUCache, TestTolerantUpdateProcessorCloud, TestAnalyzedSuggestions, TestDefaultStatsCache, TestSolrDeletionPolicy1, TestSolrXml, TestFunctionQuery, TestGraphTermsQParserPlugin, QueryEqualityTest, TestSolrQueryParserDefaultOperatorResource, DistributedFacetPivotWhiteBoxTest, FileUtilsTest, TestStandardQParsers, JavabinLoaderTest, CloudExitableDirectoryReaderTest, TestSchemaResource, TestPostingsSolrHighlighter, ShufflingReplicaListTransformerTest, DateRangeFieldTest, TestFieldCacheVsDocValues, DistributedQueryElevationComponentTest, TestConfigSetImmutable, CloneFieldUpdateProcessorFactoryTest, ShardRoutingTest, TestCollationFieldDocValues, TestStressCloudBlindAtomicUpdates, TestCollectionAPI, TestSizeLimitedDistributedMap, TestExportWriter, TestFieldTypeResource, DocumentBuilderTest, TestHighlightDedupGrouping, ShardSplitTest, ReplaceNodeTest, TestHashPartitioner, HLLSerializationTest, SynonymTokenizerTest, TestConfigSetsAPIExclusivity, UUIDUpdateProcessorFallbackTest, CachingDirectoryFactoryTest, AnalysisAfterCoreReloadTest, TestSweetSpotSimilarityFactory, DirectSolrConnectionTest, OverseerTaskQueueTest, TestNumericTerms64, DisMaxRequestHandlerTest, CdcrUpdateLogTest, TestRestManager, LeaderElectionTest, TestCustomStream, TestCloudPivotFacet, TestCryptoKeys, MergeStrategyTest, UtilsToolTest, TestSimpleQParserPlugin, TestMissingGroups, TestTrieFacet, ResponseHeaderTest, JSONWriterTest, HdfsSyncSliceTest, TestSort, TestBackupRepositoryFactory, StandardRequestHandlerTest, TestPartialUpdateDeduplication, TestElisionMultitermQuery, BJQParserTest, RecoveryAfterSoftCommitTest, TestDocumentBuilder, LeaderFailoverAfterPartitionTest, TestNamedUpdateProcessors, TestSQLHandler, AssignTest, SolrPluginUtilsTest, TestDynamicFieldResource, TestReloadDeadlock, SyncSliceTest, SolrRequestParserTest, BlockDirectoryTest, WrapperMergePolicyFactoryTest, TestCursorMarkWithoutUniqueKey, CustomCollectionTest, TimeZoneUtilsTest, PreAnalyzedFieldManagedSchemaCloudTest, TestLeaderInitiatedRecoveryThread, HardAutoCommitTest, HdfsDirectoryTest, TestCSVLoader, TestManagedSynonymFilterFactory, XsltUpdateRequestHandlerTest, StatsReloadRaceTest, BlockJoinFacetRandomTest, TestRandomDVFaceting, HdfsWriteToMultipleCollectionsTest, PluginInfoTest, TestLRUStatsCache, SolrCLIZkUtilsTest, DeleteStatusTest, CoreAdminCreateDiscoverTest, SystemInfoHandlerTest, TestGraphMLResponseWriter, TestExtendedDismaxParser, TestInitParams, PeerSyncWithIndexFingerprintCachingTest, RecoveryZkTest, TestReqParamsAPI, HdfsDirectoryFactoryTest, ClassificationUpdateProcessorIntegrationTest, TestBinaryResponseWriter, HdfsChaosMonkeySafeLeaderTest, TestBinaryField, TestHdfsBackupRestoreCore, TestSha256AuthenticationProvider, StatelessScriptUpdateProcessorFactoryTest, SolrCoreTest, TestSolrCloudWithKerberosAlt, TestTolerantSearch, ZkStateWriterTest, UniqFieldsUpdateProcessorFactoryTest, DOMUtilTest, IndexSchemaRuntimeFieldTest, TestPerFieldSimilarity, OverseerTest, TestSolrIndexConfig, ZkNodePropsTest, SchemaVersionSpecificBehaviorTest, FullSolrCloudDistribCmdsTest, TestRandomFaceting, ZkSolrClientTest, AutoCommitTest, TestDistributedGrouping, TestFaceting, TermVectorComponentDistributedTest, ZkControllerTest, TestRealTimeGet, TestStressReorder, TestJoin, TestReload, TestStressVersions, DistributedTermsComponentTest, TestCoreContainer, SpellCheckComponentTest, TestGroupingSearch, SolrCmdDistributorTest, QueryElevationComponentTest, PeerSyncTest, BadIndexSchemaTest, ConvertedLegacyTest, BasicFunctionalityTest, DirectUpdateHandlerTest, TestIndexSearcher, HighlighterTest, SoftAutoCommitTest, ShowFileRequestHandlerTest, CurrencyFieldXmlFileTest, SolrIndexSplitterTest, SuggesterFSTTest, CoreAdminHandlerTest, DocValuesTest, SuggesterTSTTest, TestStressLucene, PolyFieldTest, WordBreakSolrSpellCheckerTest, SolrCoreCheckLockOnStartupTest, TestUpdate, TestWordDelimiterFilterFactory, DocValuesMultiTest, SolrInfoMBeanTest, TestSurroundQueryParser, TestQueryUtils, DirectSolrSpellCheckerTest, DocumentAnalysisRequestHandlerTest, TestOmitPositions, PathHierarchyTokenizerFactoryTest, RequiredFieldsTest, FastVectorHighlighterTest, TestJmxIntegration, JsonLoaderTest, TestComponentsName, TestBinaryResponseWriter, TestLFUCache, PreAnalyzedFieldTest, TestSystemIdResolver, PrimUtilsTest, DateFieldTest, SpellingQueryConverterTest, RAMDirectoryFactoryTest, TestSolrJ, TestUtils, UUIDFieldTest, CircularListTest, DistributedMLTComponentTest, TestRTGBase, SolrTestCaseJ4Test, TestDistributedMissingSort, TestJettySolrRunner, ActionThrottleTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrBootstrapTest, CollectionTooManyReplicasTest, CollectionsAPISolrJTest, DeleteShardTest, DistributedQueueTest, DistributedVersionInfoTest, LeaderElectionContextKeyTest, LeaderInitiatedRecoveryOnCommitTest, LeaderInitiatedRecoveryOnShardRestartTest, MigrateRouteKeyTest, MultiThreadedOCPTest, OverseerModifyCollectionTest, RestartWhileUpdatingTest, RollingRestartTest, SSLMigrationTest, SaslZkACLProviderTest, SharedFSAutoReplicaFailoverUtilsTest, SimpleCollectionCreateDeleteTest, TestCloudRecovery, TestExclusionRuleCollectionAccess, TestLeaderElectionZkExpiry, TestLocalFSCloudBackupRestore, TestMiniSolrCloudClusterSSL, TestOnReconnectListenerSupport, TestRebalanceLeaders, TestSegmentSorting, TestShortCircuitedRequests, TestStressLiveNodes, HdfsBasicDistributedZk2Test, HdfsBasicDistributedZkTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsNNFailoverTest, HdfsRecoveryZkTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=HdfsRecoveryZkTest -Dtests.seed=AF8E0FECB28DF132 -Dtests.slow=true -Dtests.locale=ar-EG -Dtests.timezone=Africa/Bujumbura -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | HdfsRecoveryZkTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
[junit4] > at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
[junit4] > at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
[junit4] > at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
[junit4] > at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)
[junit4] > at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)
[junit4] > at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([AF8E0FECB28DF132]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] Completed [531/662 (2!)] on J0 in 48.94s, 1 test, 1 failure <<< FAILURES!
[...truncated 62823 lines...]
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 561 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/561/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC
2 tests failed.
FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI
Error Message:
expected:<3> but was:<1>
Stack Trace:
java.lang.AssertionError: expected:<3> but was:<1>
at __randomizedtesting.SeedInfo.seed([FC0068A0A00BD094:B4751C14A638FF01]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:516)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130) at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102) at sun.reflect.GeneratedConstructorAccessor146.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
at sun.reflect.GeneratedConstructorAccessor146.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)
at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)
at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([FC0068A0A00BD094]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
at sun.reflect.GeneratedMethodAccessor25.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 11479 lines...]
[junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/init-core-data-001
[junit4] 2> 944393 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[FC0068A0A00BD094]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 944393 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[FC0068A0A00BD094]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001
[junit4] 2> 944394 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[FC0068A0A00BD094]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 944394 INFO (Thread-3199) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 944394 INFO (Thread-3199) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 944494 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[FC0068A0A00BD094]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:44060
[junit4] 2> 944503 INFO (jetty-launcher-1590-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 944504 INFO (jetty-launcher-1590-thread-3) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 944504 INFO (jetty-launcher-1590-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 944504 INFO (jetty-launcher-1590-thread-4) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 944524 INFO (jetty-launcher-1590-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1ff15dad{/solr,null,AVAILABLE}
[junit4] 2> 944526 INFO (jetty-launcher-1590-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6588b5ad{/solr,null,AVAILABLE}
[junit4] 2> 944526 INFO (jetty-launcher-1590-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@ad72140{/solr,null,AVAILABLE}
[junit4] 2> 944527 INFO (jetty-launcher-1590-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5b5bcbf3{/solr,null,AVAILABLE}
[junit4] 2> 944527 INFO (jetty-launcher-1590-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@79b99274{HTTP/1.1,[http/1.1]}{127.0.0.1:54744}
[junit4] 2> 944527 INFO (jetty-launcher-1590-thread-4) [ ] o.e.j.s.Server Started @948708ms
[junit4] 2> 944528 INFO (jetty-launcher-1590-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54744}
[junit4] 2> 944528 ERROR (jetty-launcher-1590-thread-4) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 944529 INFO (jetty-launcher-1590-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@10926bbc{HTTP/1.1,[http/1.1]}{127.0.0.1:51783}
[junit4] 2> 944529 INFO (jetty-launcher-1590-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.4.0
[junit4] 2> 944529 INFO (jetty-launcher-1590-thread-3) [ ] o.e.j.s.Server Started @948710ms
[junit4] 2> 944529 INFO (jetty-launcher-1590-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 944529 INFO (jetty-launcher-1590-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51783}
[junit4] 2> 944529 INFO (jetty-launcher-1590-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 944529 INFO (jetty-launcher-1590-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-20T00:35:43.668Z
[junit4] 2> 944529 ERROR (jetty-launcher-1590-thread-3) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 944529 INFO (jetty-launcher-1590-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.4.0
[junit4] 2> 944529 INFO (jetty-launcher-1590-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7b95d0a8{HTTP/1.1,[http/1.1]}{127.0.0.1:48967}
[junit4] 2> 944529 INFO (jetty-launcher-1590-thread-1) [ ] o.e.j.s.Server Started @948710ms
[junit4] 2> 944529 INFO (jetty-launcher-1590-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=48967}
[junit4] 2> 944529 ERROR (jetty-launcher-1590-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 944529 INFO (jetty-launcher-1590-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 944530 INFO (jetty-launcher-1590-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.4.0
[junit4] 2> 944530 INFO (jetty-launcher-1590-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 944530 INFO (jetty-launcher-1590-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 944530 INFO (jetty-launcher-1590-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-20T00:35:43.669Z
[junit4] 2> 944530 INFO (jetty-launcher-1590-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 944530 INFO (jetty-launcher-1590-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-20T00:35:43.669Z
[junit4] 2> 944530 INFO (jetty-launcher-1590-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5d916031{HTTP/1.1,[http/1.1]}{127.0.0.1:59668}
[junit4] 2> 944530 INFO (jetty-launcher-1590-thread-2) [ ] o.e.j.s.Server Started @948711ms
[junit4] 2> 944531 INFO (jetty-launcher-1590-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59668}
[junit4] 2> 944531 ERROR (jetty-launcher-1590-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 944531 INFO (jetty-launcher-1590-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.4.0
[junit4] 2> 944531 INFO (jetty-launcher-1590-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 944531 INFO (jetty-launcher-1590-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 944531 INFO (jetty-launcher-1590-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-20T00:35:43.670Z
[junit4] 2> 944535 INFO (jetty-launcher-1590-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 944536 INFO (jetty-launcher-1590-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 944540 INFO (jetty-launcher-1590-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 944542 INFO (jetty-launcher-1590-thread-4) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 944542 INFO (jetty-launcher-1590-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 944543 INFO (jetty-launcher-1590-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 944543 INFO (jetty-launcher-1590-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44060/solr
[junit4] 2> 944549 INFO (jetty-launcher-1590-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44060/solr
[junit4] 2> 944555 INFO (jetty-launcher-1590-thread-3) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 944556 INFO (jetty-launcher-1590-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44060/solr
[junit4] 2> 944557 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15919a7c92e0003, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 944557 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15919a7c92e0002, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 944561 INFO (jetty-launcher-1590-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 944562 INFO (jetty-launcher-1590-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44060/solr
[junit4] 2> 944595 INFO (jetty-launcher-1590-thread-4) [n:127.0.0.1:54744_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:54744_solr
[junit4] 2> 944597 INFO (jetty-launcher-1590-thread-4) [n:127.0.0.1:54744_solr ] o.a.s.c.Overseer Overseer (id=97137075390644232-127.0.0.1:54744_solr-n_0000000000) starting
[junit4] 2> 944606 INFO (jetty-launcher-1590-thread-4) [n:127.0.0.1:54744_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54744_solr
[junit4] 2> 944632 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15919a7c92e0009, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 944644 INFO (jetty-launcher-1590-thread-2) [n:127.0.0.1:59668_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 944655 INFO (jetty-launcher-1590-thread-2) [n:127.0.0.1:59668_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59668_solr
[junit4] 2> 944662 INFO (zkCallback-1609-thread-1-processing-n:127.0.0.1:59668_solr) [n:127.0.0.1:59668_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 944675 INFO (jetty-launcher-1590-thread-1) [n:127.0.0.1:48967_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 944676 INFO (jetty-launcher-1590-thread-3) [n:127.0.0.1:51783_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 944682 INFO (jetty-launcher-1590-thread-1) [n:127.0.0.1:48967_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48967_solr
[junit4] 2> 944682 INFO (jetty-launcher-1590-thread-3) [n:127.0.0.1:51783_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51783_solr
[junit4] 2> 944692 INFO (zkCallback-1608-thread-1-processing-n:127.0.0.1:54744_solr) [n:127.0.0.1:54744_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 944708 INFO (zkCallback-1609-thread-1-processing-n:127.0.0.1:59668_solr) [n:127.0.0.1:59668_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
[junit4] 2> 944722 INFO (OverseerStateUpdate-97137075390644232-127.0.0.1:54744_solr-n_0000000000) [n:127.0.0.1:54744_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
[junit4] 2> 944738 INFO (zkCallback-1613-thread-1-processing-n:127.0.0.1:48967_solr) [n:127.0.0.1:48967_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
[junit4] 2> 944739 INFO (zkCallback-1612-thread-1-processing-n:127.0.0.1:51783_solr) [n:127.0.0.1:51783_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
[junit4] 2> 944743 INFO (jetty-launcher-1590-thread-1) [n:127.0.0.1:48967_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node1/.
[junit4] 2> 944775 INFO (jetty-launcher-1590-thread-2) [n:127.0.0.1:59668_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node2/.
[junit4] 2> 944779 INFO (jetty-launcher-1590-thread-4) [n:127.0.0.1:54744_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node3/.
[junit4] 2> 944941 INFO (jetty-launcher-1590-thread-3) [n:127.0.0.1:51783_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node4/.
[junit4] 2> 944949 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[FC0068A0A00BD094]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 944951 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[FC0068A0A00BD094]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44060/solr ready
[junit4] 2> 945000 INFO (TEST-CollectionsAPIDistributedZkTest.testTooManyReplicas-seed#[FC0068A0A00BD094]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testTooManyReplicas
[junit4] 2> 945002 INFO (TEST-CollectionsAPIDistributedZkTest.testTooManyReplicas-seed#[FC0068A0A00BD094]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 945009 INFO (qtp1374301387-10278) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=10&collection.configName=conf&name=collection&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 945014 INFO (OverseerThreadFactory-5021-thread-1-processing-n:127.0.0.1:54744_solr) [n:127.0.0.1:54744_solr ] o.a.s.c.CreateCollectionCmd Create collection collection
[junit4] 2> 945015 WARN (OverseerThreadFactory-5021-thread-1-processing-n:127.0.0.1:54744_solr) [n:127.0.0.1:54744_solr ] o.a.s.c.CreateCollectionCmd Specified replicationFactor of 10 on collection collection is higher than or equal to the number of Solr instances currently live or live and part of your createNodeSet(4). It's unusual to run two replica of the same slice on the same Solr-instance.
[junit4] 2> 945015 ERROR (OverseerThreadFactory-5021-thread-1-processing-n:127.0.0.1:54744_solr) [n:127.0.0.1:54744_solr ] o.a.s.c.OverseerCollectionMessageHandler Collection: collection operation: create failed:org.apache.solr.common.SolrException: Cannot create collection collection. Value of maxShardsPerNode is 1, and the number of nodes currently live or live and part of your createNodeSet is 4. This allows a maximum of 4 to be created. Value of numShards is 2 and value of replicationFactor is 10. This requires 20 shards to be created (higher than the allowed number)
[junit4] 2> at org.apache.solr.cloud.CreateCollectionCmd.call(CreateCollectionCmd.java:155)
[junit4] 2> at org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:224)
[junit4] 2> at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:463)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 945019 INFO (qtp1374301387-10278) [n:127.0.0.1:48967_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=10&collection.configName=conf&name=collection&action=CREATE&numShards=2&wt=javabin&version=2} status=400 QTime=10
[junit4] 2> 945020 INFO (TEST-CollectionsAPIDistributedZkTest.testTooManyReplicas-seed#[FC0068A0A00BD094]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testTooManyReplicas
[junit4] 2> 945055 INFO (TEST-CollectionsAPIDistributedZkTest.testNoCollectionSpecified-seed#[FC0068A0A00BD094]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNoCollectionSpecified
[junit4] 2> 945057 INFO (TEST-CollectionsAPIDistributedZkTest.testNoCollectionSpecified-seed#[FC0068A0A00BD094]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 945065 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CoreAdminOperation core create command dataDir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-002&collection.configName=conf&name=corewithnocollection&action=CREATE&numShards=1&collection=&wt=javabin&version=2
[junit4] 2> 945068 INFO (OverseerStateUpdate-97137075390644232-127.0.0.1:54744_solr-n_0000000000) [n:127.0.0.1:54744_solr ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 946081 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 946099 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.s.IndexSchema [corewithnocollection] Schema name=minimal
[junit4] 2> 946099 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 946105 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.CoreContainer Creating SolrCore 'corewithnocollection' using configuration from collection corewithnocollection
[junit4] 2> 946106 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.SolrCore [[corewithnocollection] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node1/corewithnocollection], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-002/]
[junit4] 2> 946115 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 946115 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 946116 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 946116 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 946117 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.s.SolrIndexSearcher Opening [Searcher@259f0294[corewithnocollection] main]
[junit4] 2> 946119 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 946119 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 946119 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 946120 INFO (searcherExecutor-5030-thread-1-processing-n:127.0.0.1:48967_solr x:corewithnocollection c:corewithnocollection) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.SolrCore [corewithnocollection] Registered new searcher Searcher@259f0294[corewithnocollection] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 946120 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1554193208059101184
[junit4] 2> 946130 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 946130 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 946130 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:48967/solr/corewithnocollection/
[junit4] 2> 946130 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 946130 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.SyncStrategy http://127.0.0.1:48967/solr/corewithnocollection/ has no replicas
[junit4] 2> 946138 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:48967/solr/corewithnocollection/ shard1
[junit4] 2> 946297 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 946300 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection x:corewithnocollection] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={dataDir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-002&collection.configName=conf&name=corewithnocollection&action=CREATE&numShards=1&collection=&wt=javabin&version=2} status=0 QTime=1236
[junit4] 2> 946302 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CoreAdminOperation core create command dataDir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-002&collection.configName=conf&name=corewithnocollection2&action=CREATE&numShards=1&wt=javabin&version=2
[junit4] 2> 946305 INFO (OverseerStateUpdate-97137075390644232-127.0.0.1:54744_solr-n_0000000000) [n:127.0.0.1:54744_solr ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 947020 INFO (OverseerCollectionConfigSetProcessor-97137075390644232-127.0.0.1:54744_solr-n_0000000000) [n:127.0.0.1:54744_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 947323 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 947332 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.s.IndexSchema [corewithnocollection2] Schema name=minimal
[junit4] 2> 947335 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 947336 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.CoreContainer Creating SolrCore 'corewithnocollection2' using configuration from collection corewithnocollection2
[junit4] 2> 947336 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.SolrCore [[corewithnocollection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node1/corewithnocollection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-002/]
[junit4] 2> 947346 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 947346 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 947346 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 947346 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 947347 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@8e92225[corewithnocollection2] main]
[junit4] 2> 947349 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 947349 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 947349 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 947349 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1554193209347801088
[junit4] 2> 947349 INFO (searcherExecutor-5035-thread-1-processing-n:127.0.0.1:48967_solr x:corewithnocollection2 c:corewithnocollection2) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.SolrCore [corewithnocollection2] Registered new searcher Searcher@8e92225[corewithnocollection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 947359 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 947359 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 947359 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:48967/solr/corewithnocollection2/
[junit4] 2> 947359 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 947359 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.SyncStrategy http://127.0.0.1:48967/solr/corewithnocollection2/ has no replicas
[junit4] 2> 947367 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:48967/solr/corewithnocollection2/ shard1
[junit4] 2> 947519 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 947523 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr c:corewithnocollection2 x:corewithnocollection2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={dataDir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-002&collection.configName=conf&name=corewithnocollection2&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=1220
[junit4] 2> 947524 INFO (TEST-CollectionsAPIDistributedZkTest.testNoCollectionSpecified-seed#[FC0068A0A00BD094]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNoCollectionSpecified
[junit4] 2> 947563 INFO (TEST-CollectionsAPIDistributedZkTest.deleteCollectionOnlyInZk-seed#[FC0068A0A00BD094]) [ ] o.a.s.SolrTestCaseJ4 ###Starting deleteCollectionOnlyInZk
[junit4] 2> 947565 INFO (TEST-CollectionsAPIDistributedZkTest.deleteCollectionOnlyInZk-seed#[FC0068A0A00BD094]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 947569 INFO (qtp1374301387-10288) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=corewithnocollection&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 947573 INFO (OverseerThreadFactory-5021-thread-2-processing-n:127.0.0.1:54744_solr) [n:127.0.0.1:54744_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 947576 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr ] o.a.s.c.SolrCore [corewithnocollection] CLOSING SolrCore org.apache.solr.core.SolrCore@3501eee4
[junit4] 2> 947579 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=corewithnocollection&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 948287 INFO (qtp1374301387-10288) [n:127.0.0.1:48967_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=corewithnocollection&action=DELETE&wt=javabin&version=2} status=0 QTime=718
[junit4] 2> 948289 INFO (qtp1374301387-10268) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=corewithnocollection2&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 948294 INFO (OverseerThreadFactory-5021-thread-3-processing-n:127.0.0.1:54744_solr) [n:127.0.0.1:54744_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 948294 INFO (OverseerCollectionConfigSetProcessor-97137075390644232-127.0.0.1:54744_solr-n_0000000000) [n:127.0.0.1:54744_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 948296 INFO (qtp1374301387-10278) [n:127.0.0.1:48967_solr ] o.a.s.c.SolrCore [corewithnocollection2] CLOSING SolrCore org.apache.solr.core.SolrCore@7f143aec
[junit4] 2> 948300 INFO (qtp1374301387-10278) [n:127.0.0.1:48967_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=corewithnocollection2&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=4
[junit4] 2> 949014 INFO (qtp1374301387-10268) [n:127.0.0.1:48967_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=corewithnocollection2&action=DELETE&wt=javabin&version=2} status=0 QTime=725
[junit4] 2> 949022 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=onlyinzk&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 949026 INFO (OverseerCollectionConfigSetProcessor-97137075390644232-127.0.0.1:54744_solr-n_0000000000) [n:127.0.0.1:54744_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 949034 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=onlyinzk&action=DELETE&wt=javabin&version=2} status=0 QTime=11
[junit4] 2> 949041 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 949041 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 949043 INFO (qtp1374301387-10289) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=onlyinzk&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 949048 INFO (OverseerThreadFactory-5021-thread-5-processing-n:127.0.0.1:54744_solr) [n:127.0.0.1:54744_solr ] o.a.s.c.CreateCollectionCmd Create collection onlyinzk
[junit4] 2> 949048 INFO (OverseerCollectionConfigSetProcessor-97137075390644232-127.0.0.1:54744_solr-n_0000000000) [n:127.0.0.1:54744_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 949256 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=onlyinzk_shard2_replica1&action=CREATE&numShards=2&collection=onlyinzk&shard=shard2&wt=javabin&version=2
[junit4] 2> 949257 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=onlyinzk_shard1_replica1&action=CREATE&numShards=2&collection=onlyinzk&shard=shard1&wt=javabin&version=2
[junit4] 2> 949367 INFO (zkCallback-1613-thread-1-processing-n:127.0.0.1:48967_solr) [n:127.0.0.1:48967_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/onlyinzk/state.json] for collection [onlyinzk] has occurred - updating... (live nodes size: [4])
[junit4] 2> 949367 INFO (zkCallback-1612-thread-1-processing-n:127.0.0.1:51783_solr) [n:127.0.0.1:51783_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/onlyinzk/state.json] for collection [onlyinzk] has occurred - updating... (live nodes size: [4])
[junit4] 2> 950272 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 950298 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.s.IndexSchema [onlyinzk_shard2_replica1] Schema name=minimal
[junit4] 2> 950300 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 950302 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.c.CoreContainer Creating SolrCore 'onlyinzk_shard2_replica1' using configuration from collection onlyinzk
[junit4] 2> 950302 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.c.SolrCore [[onlyinzk_shard2_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node4/onlyinzk_shard2_replica1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node4/./onlyinzk_shard2_replica1/data/]
[junit4] 2> 950305 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 950313 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.s.IndexSchema [onlyinzk_shard1_replica1] Schema name=minimal
[junit4] 2> 950314 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 950314 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 950315 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 950315 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 950315 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 950317 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6a7ea0d0[onlyinzk_shard2_replica1] main]
[junit4] 2> 950317 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'onlyinzk_shard1_replica1' using configuration from collection onlyinzk
[junit4] 2> 950318 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.c.SolrCore [[onlyinzk_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node1/onlyinzk_shard1_replica1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node1/./onlyinzk_shard1_replica1/data/]
[junit4] 2> 950318 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 950319 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 950319 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 950319 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1554193212462071808
[junit4] 2> 950325 INFO (searcherExecutor-5040-thread-1-processing-n:127.0.0.1:51783_solr x:onlyinzk_shard2_replica1 s:shard2 c:onlyinzk) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.c.SolrCore [onlyinzk_shard2_replica1] Registered new searcher Searcher@6a7ea0d0[onlyinzk_shard2_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 950326 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 950326 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 950327 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 950327 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 950327 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 950327 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 950327 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:51783/solr/onlyinzk_shard2_replica1/
[junit4] 2> 950327 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@31e5b5b5[onlyinzk_shard1_replica1] main]
[junit4] 2> 950327 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 950327 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:51783/solr/onlyinzk_shard2_replica1/ has no replicas
[junit4] 2> 950329 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 950329 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 950329 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 950329 INFO (searcherExecutor-5045-thread-1-processing-n:127.0.0.1:48967_solr x:onlyinzk_shard1_replica1 s:shard1 c:onlyinzk) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.c.SolrCore [onlyinzk_shard1_replica1] Registered new searcher Searcher@31e5b5b5[onlyinzk_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 950329 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1554193212472557568
[junit4] 2> 950333 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:51783/solr/onlyinzk_shard2_replica1/ shard2
[junit4] 2> 950337 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 950337 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 950337 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:48967/solr/onlyinzk_shard1_replica1/
[junit4] 2> 950337 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 950337 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:48967/solr/onlyinzk_shard1_replica1/ has no replicas
[junit4] 2> 950342 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:48967/solr/onlyinzk_shard1_replica1/ shard1
[junit4] 2> 950450 INFO (zkCallback-1613-thread-1-processing-n:127.0.0.1:48967_solr) [n:127.0.0.1:48967_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/onlyinzk/state.json] for collection [onlyinzk] has occurred - updating... (live nodes size: [4])
[junit4] 2> 950451 INFO (zkCallback-1612-thread-1-processing-n:127.0.0.1:51783_solr) [n:127.0.0.1:51783_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/onlyinzk/state.json] for collection [onlyinzk] has occurred - updating... (live nodes size: [4])
[junit4] 2> 950485 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 950488 INFO (qtp916004545-10270) [n:127.0.0.1:51783_solr c:onlyinzk s:shard2 x:onlyinzk_shard2_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=onlyinzk_shard2_replica1&action=CREATE&numShards=2&collection=onlyinzk&shard=shard2&wt=javabin&version=2} status=0 QTime=1232
[junit4] 2> 950493 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 950497 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr c:onlyinzk s:shard1 x:onlyinzk_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=onlyinzk_shard1_replica1&action=CREATE&numShards=2&collection=onlyinzk&shard=shard1&wt=javabin&version=2} status=0 QTime=1240
[junit4] 2> 950501 INFO (qtp1374301387-10289) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 950603 INFO (zkCallback-1613-thread-1-processing-n:127.0.0.1:48967_solr) [n:127.0.0.1:48967_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/onlyinzk/state.json] for collection [onlyinzk] has occurred - updating... (live nodes size: [4])
[junit4] 2> 950603 INFO (zkCallback-1612-thread-1-processing-n:127.0.0.1:51783_solr) [n:127.0.0.1:51783_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/onlyinzk/state.json] for collection [onlyinzk] has occurred - updating... (live nodes size: [4])
[junit4] 2> 951051 INFO (OverseerCollectionConfigSetProcessor-97137075390644232-127.0.0.1:54744_solr-n_0000000000) [n:127.0.0.1:54744_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000008 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 951502 INFO (qtp1374301387-10289) [n:127.0.0.1:48967_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=onlyinzk&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=2460
[junit4] 2> 951505 INFO (qtp1374301387-10288) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 951505 INFO (qtp1374301387-10288) [n:127.0.0.1:48967_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 951505 INFO (TEST-CollectionsAPIDistributedZkTest.deleteCollectionOnlyInZk-seed#[FC0068A0A00BD094]) [ ] o.a.s.SolrTestCaseJ4 ###Ending deleteCollectionOnlyInZk
[junit4] 2> 951549 INFO (TEST-CollectionsAPIDistributedZkTest.testNoConfigSetExist-seed#[FC0068A0A00BD094]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNoConfigSetExist
[junit4] 2> 951552 INFO (TEST-CollectionsAPIDistributedZkTest.testNoConfigSetExist-seed#[FC0068A0A00BD094]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 951556 INFO (qtp1374301387-10278) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=onlyinzk&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 951562 INFO (OverseerThreadFactory-5021-thread-5-processing-n:127.0.0.1:54744_solr) [n:127.0.0.1:54744_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 951567 INFO (qtp1374301387-10268) [n:127.0.0.1:48967_solr ] o.a.s.c.SolrCore [onlyinzk_shard1_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@da007f9
[junit4] 2> 951570 INFO (qtp916004545-10272) [n:127.0.0.1:51783_solr ] o.a.s.c.SolrCore [onlyinzk_shard2_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@765e51b5
[junit4] 2> 951572 INFO (qtp1374301387-10268) [n:127.0.0.1:48967_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=onlyinzk_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=4
[junit4] 2> 951579 INFO (qtp916004545-10272) [n:127.0.0.1:51783_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=onlyinzk_shard2_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=9
[junit4] 2> 952288 INFO (qtp1374301387-10278) [n:127.0.0.1:48967_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=onlyinzk&action=DELETE&wt=javabin&version=2} status=0 QTime=733
[junit4] 2> 952291 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CoreAdminOperation core create command dataDir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-003&collection.configName=conf123&name=corewithnocollection3&action=CREATE&numShards=1&collection=&wt=javabin&version=2
[junit4] 2> 952293 INFO (OverseerStateUpdate-97137075390644232-127.0.0.1:54744_solr-n_0000000000) [n:127.0.0.1:54744_solr ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 953299 ERROR (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection3 x:corewithnocollection3] o.a.s.c.c.ZkStateReader Specified config=[conf123] does not exist in ZooKeeper at location=[/configs/conf123]
[junit4] 2> 953300 ERROR (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection3 x:corewithnocollection3] o.a.s.c.CoreContainer Error creating core [corewithnocollection3]: Specified config does not exist in ZooKeeper: conf123
[junit4] 2> org.apache.solr.common.cloud.ZooKeeperException: Specified config does not exist in ZooKeeper: conf123
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.readConfigName(ZkStateReader.java:189)
[junit4] 2> at org.apache.solr.cloud.CloudConfigSetService.createCoreResourceLoader(CloudConfigSetService.java:54)
[junit4] 2> at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:840)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:779)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:88)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:377)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:365)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:156)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:152)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:664)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:445)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:318)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:269)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:110)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 953301 ERROR (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection3 x:corewithnocollection3] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'corewithnocollection3': Unable to create core [corewithnocollection3] Caused by: Specified config does not exist in ZooKeeper: conf123
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:813)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:88)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:377)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:365)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:156)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:152)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:664)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:445)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:318)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:269)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:110)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: org.apache.solr.common.SolrException: Unable to create core [corewithnocollection3]
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:855)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:779)
[junit4] 2> ... 33 more
[junit4] 2> Caused by: org.apache.solr.common.cloud.ZooKeeperException: Specified config does not exist in ZooKeeper: conf123
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.readConfigName(ZkStateReader.java:189)
[junit4] 2> at org.apache.solr.cloud.CloudConfigSetService.createCoreResourceLoader(CloudConfigSetService.java:54)
[junit4] 2> at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:840)
[junit4] 2> ... 34 more
[junit4] 2>
[junit4] 2> 953301 INFO (qtp1374301387-10280) [n:127.0.0.1:48967_solr c:corewithnocollection3 x:corewithnocollection3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={dataDir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-003&collection.configName=conf123&name=corewithnocollection3&action=CREATE&numShards=1&collection=&wt=javabin&version=2} status=400 QTime=1010
[junit4] 2> 953505 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 953505 INFO (qtp1374301387-10287) [n:127.0.0.1:48967_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 953505 INFO (TEST-CollectionsAPIDistributedZkTest.testNoConfigSetExist-seed#[FC0068A0A00BD094]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNoConfigSetExist
[junit4] 2> 953556 INFO (TEST-CollectionsAPIDistributedZkTest.testMaxNodesPerShard-seed#[FC0068A0A00BD094]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testMaxNodesPerShard
[junit4] 2> 953558 INFO (TEST-CollectionsAPIDistributedZkTest.testMaxNodesPerShard-seed#[FC0068A0A00BD094]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 953562 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=corewithnocollection3&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 953565 INFO (OverseerThreadFactory-5021-thread-5-processing-n:127.0.0.1:54744_solr) [n:127.0.0.1:54744_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 953566 INFO (OverseerCollectionConfigSetProcessor-97137075390644232-127.0.0.1:54744_solr-n_0000000000) [n:127.0.0.1:54744_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000010 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 954275 INFO (qtp1374301387-10290) [n:127.0.0.1:48967_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=corewithnocollection3&action=DELETE&wt=javabin&version=2} status=0 QTime=713
[junit4] 2> 954279 INFO (qtp1374301387-10289) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&name=oversharded&action=CREATE&numShards=3&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 954285 INFO (OverseerCollectionConfigSetProcessor-97137075390644232-127.0.0.1:54744_solr-n_0000000000) [n:127.0.0.1:54744_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000012 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 954286 INFO (OverseerThreadFactory-5021-thread-5-processing-n:127.0.0.1:54744_solr) [n:127.0.0.1:54744_solr ] o.a.s.c.CreateCollectionCmd Create collection oversharded
[junit4] 2> 954286 ERROR (OverseerThreadFactory-5021-thread-5-processing-n:127.0.0.1:54744_solr) [n:127.0.0.1:54744_solr ] o.a.s.c.OverseerCollectionMessageHandler Collection: oversharded operation: create failed:org.apache.solr.common.SolrException: Cannot create collection oversharded. Value of maxShardsPerNode is 1, and the number of nodes currently live or live and part of your createNodeSet is 4. This allows a maximum of 4 to be created. Value of numShards is 3 and value of replicationFactor is 2. This requires 6 shards to be created (higher than the allowed number)
[junit4] 2> at org.apache.solr.cloud.CreateCollectionCmd.call(CreateCollectionCmd.java:155)
[junit4] 2> at org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:224)
[junit4] 2> at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:463)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 954291 INFO (qtp1374301387-10289) [n:127.0.0.1:48967_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&name=oversharded&action=CREATE&numShards=3&wt=javabin&version=2} status=400 QTime=11
[junit4] 2> 954291 INFO (TEST-CollectionsAPIDistributedZkTest.testMaxNodesPerShard-seed#[FC0068A0A00BD094]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testMaxNodesPerShard
[junit4] 2> 954349 INFO (TEST-CollectionsAPIDistributedZkTest.addReplicaTest-seed#[FC0068A0A00BD094]) [ ] o.a.s.SolrTestCaseJ4 ###Starting addReplicaTest
[junit4] 2> 954351 INFO (TEST-CollectionsAPIDistributedZkTest.addReplicaTest-seed#[FC0068A0A00BD094]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 954355 INFO (qtp1374301387-10288) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=4&name=addReplicaColl&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 954358 INFO (OverseerThreadFactory-5021-thread-5-processing-n:127.0.0.1:54744_solr) [n:127.0.0.1:54744_solr ] o.a.s.c.CreateCollectionCmd Create collection addReplicaColl
[junit4] 2> 954358 INFO (OverseerCollectionConfigSetProcessor-97137075390644232-127.0.0.1:54744_solr-n_0000000000) [n:127.0.0.1:54744_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000014 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 954573 INFO (qtp916004545-10272) [n:127.0.0.1:51783_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=addReplicaColl_shard1_replica2&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard1&wt=javabin&version=2
[junit4] 2> 954577 INFO (qtp1374301387-10268) [n:127.0.0.1:48967_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=addReplicaColl_shard2_replica2&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard2&wt=javabin&version=2
[junit4] 2> 954578 INFO (qtp1538009533-10267) [n:127.0.0.1:59668_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=addReplicaColl_shard2_replica1&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard2&wt=javabin&version=2
[junit4] 2> 954579 INFO (qtp27260732-10293) [n:127.0.0.1:54744_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=addReplicaColl_shard1_replica1&action=CREATE&numShards=2&collection=addReplicaColl&shard=shard1&wt=javabin&version=2
[junit4] 2> 954700 INFO (zkCallback-1613-thread-2-processing-n:127.0.0.1:48967_solr) [n:127.0.0.1:48967_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
[junit4] 2> 954700 INFO (zkCallback-1608-thread-1-processing-n:127.0.0.1:54744_solr) [n:127.0.0.1:54744_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
[junit4] 2> 954700 INFO (zkCallback-1609-thread-1-processing-n:127.0.0.1:59668_solr) [n:127.0.0.1:59668_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
[junit4] 2> 954700 INFO (zkCallback-1612-thread-2-processing-n:127.0.0.1:51783_solr) [n:127.0.0.1:51783_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/addReplicaColl/state.json] for collection [addReplicaColl] has occurred - updating... (live nodes size: [4])
[junit4] 2> 955593 INFO (qtp27260732-10293) [n:127.0.0.1:54744_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 955594 INFO (qtp1374301387-10268) [n:127.0.0.1:48967_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 955597 INFO (qtp916004545-10272) [n:127.0.0.1:51783_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 955614 INFO (qtp1538009533-10267) [n:127.0.0.1:59668_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 955617 INFO (qtp916004545-10272) [n:127.0.0.1:51783_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.s.IndexSchema [addReplicaColl_shard1_replica2] Schema name=minimal
[junit4] 2> 955621 INFO (qtp916004545-10272) [n:127.0.0.1:51783_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 955622 INFO (qtp916004545-10272) [n:127.0.0.1:51783_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'addReplicaColl_shard1_replica2' using configuration from collection addReplicaColl
[junit4] 2> 955622 INFO (qtp916004545-10272) [n:127.0.0.1:51783_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.c.SolrCore [[addReplicaColl_shard1_replica2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node4/addReplicaColl_shard1_replica2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node4/./addReplicaColl_shard1_replica2/data/]
[junit4] 2> 955623 INFO (qtp27260732-10293) [n:127.0.0.1:54744_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica1] o.a.s.s.IndexSchema [addReplicaColl_shard1_replica1] Schema name=minimal
[junit4] 2> 955623 INFO (qtp1374301387-10268) [n:127.0.0.1:48967_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica2] o.a.s.s.IndexSchema [addReplicaColl_shard2_replica2] Schema name=minimal
[junit4] 2> 955625 INFO (qtp27260732-10293) [n:127.0.0.1:54744_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 955625 INFO (qtp1374301387-10268) [n:127.0.0.1:48967_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 955626 INFO (qtp1374301387-10268) [n:127.0.0.1:48967_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica2] o.a.s.c.CoreContainer Creating SolrCore 'addReplicaColl_shard2_replica2' using configuration from collection addReplicaColl
[junit4] 2> 955626 INFO (qtp1374301387-10268) [n:127.0.0.1:48967_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica2] o.a.s.c.SolrCore [[addReplicaColl_shard2_replica2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node1/addReplicaColl_shard2_replica2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node1/./addReplicaColl_shard2_replica2/data/]
[junit4] 2> 955627 INFO (qtp1538009533-10267) [n:127.0.0.1:59668_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica1] o.a.s.s.IndexSchema [addReplicaColl_shard2_replica1] Schema name=minimal
[junit4] 2> 955630 INFO (qtp1538009533-10267) [n:127.0.0.1:59668_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 955630 INFO (qtp27260732-10293) [n:127.0.0.1:54744_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'addReplicaColl_shard1_replica1' using configuration from collection addReplicaColl
[junit4] 2> 955631 INFO (qtp27260732-10293) [n:127.0.0.1:54744_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica1] o.a.s.c.SolrCore [[addReplicaColl_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node3/addReplicaColl_shard1_replica1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_FC0068A0A00BD094-001/tempDir-001/node3/./addReplicaColl_shard1_replica1/data/]
[junit4] 2> 955634 INFO (qtp916004545-10272) [n:127.0.0.1:51783_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 955634 INFO (qtp916004545-10272) [n:127.0.0.1:51783_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 955636 INFO (qtp916004545-10272) [n:127.0.0.1:51783_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 955636 INFO (qtp916004545-10272) [n:127.0.0.1:51783_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 955636 INFO (qtp1374301387-10268) [n:127.0.0.1:48967_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 955636 INFO (qtp1374301387-10268) [n:127.0.0.1:48967_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 955636 INFO (qtp916004545-10272) [n:127.0.0.1:51783_solr c:addReplicaColl s:shard1 x:addReplicaColl_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@2e7e63b5[addReplicaColl_shard1_replica2] main]
[junit4] 2> 955637 INFO (qtp1374301387-10268) [n:127.0.0.1:48967_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 955637 INFO (qtp1374301387-10268) [n:127.0.0.1:48967_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 955637 INFO (qtp1374301387-10268) [n:127.0.0.1:48967_solr c:addReplicaColl s:shard2 x:addReplicaColl_shard2_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@5d90dd24[addReplicaColl_shard2_replica2] main]
[junit4] 2> 95
[...truncated too long message...]
rrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> 2451572 INFO (SUITE-HdfsRecoveryZkTest-seed#[FC0068A0A00BD094]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:48005 48005
[junit4] 2> 2451573 INFO (SUITE-HdfsRecoveryZkTest-seed#[FC0068A0A00BD094]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=false,coord=yes): {}, locale=sr-Latn, timezone=America/North_Dakota/Beulah
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=2,free=157600112,total=518979584
[junit4] 2> NOTE: All tests run in this JVM: [TlogReplayBufferedWhileIndexingTest, PKIAuthenticationIntegrationTest, HdfsNNFailoverTest, TestFieldTypeResource, TestReqParamsAPI, TestCloudPivotFacet, BlobRepositoryCloudTest, BaseCdcrDistributedZkTest, TestExclusionRuleCollectionAccess, DocumentAnalysisRequestHandlerTest, AlternateDirectoryTest, TestGroupingSearch, BadComponentTest, TestCursorMarkWithoutUniqueKey, TestReplicationHandler, TestComplexPhraseQParserPlugin, ConcurrentDeleteAndCreateCollectionTest, TestBinaryResponseWriter, TestStressLucene, DistributedQueryElevationComponentTest, TestDocumentBuilder, TestPerFieldSimilarity, XsltUpdateRequestHandlerTest, HdfsWriteToMultipleCollectionsTest, AssignTest, TestComponentsName, AnalysisAfterCoreReloadTest, SpatialFilterTest, TestNamedUpdateProcessors, TestHighlightDedupGrouping, TestCollectionAPI, TestRandomCollapseQParserPlugin, TestElisionMultitermQuery, TestImplicitCoreProperties, DisMaxRequestHandlerTest, TestLRUCache, CollectionStateFormat2Test, TestLeaderInitiatedRecoveryThread, TestUniqueKeyFieldResource, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestLFUCache, OpenExchangeRatesOrgProviderTest, JavabinLoaderTest, TestLeaderElectionWithEmptyReplica, SecurityConfHandlerTest, TestPseudoReturnFields, TestLRUStatsCache, RecoveryAfterSoftCommitTest, BasicDistributedZkTest, FastVectorHighlighterTest, XmlUpdateRequestHandlerTest, TestBinaryResponseWriter, PeerSyncWithIndexFingerprintCachingTest, DocumentBuilderTest, TestBlobHandler, IndexBasedSpellCheckerTest, NumericFieldsTest, StatsReloadRaceTest, TestCollationField, TestSurroundQueryParser, DirectoryFactoryTest, TestLegacyFieldCache, TestManagedSchemaAPI, TestStressRecovery, TestClassicSimilarityFactory, TestManagedResource, CollectionsAPIAsyncDistributedZkTest, RuleEngineTest, TestLockTree, DistributedMLTComponentTest, SharedFSAutoReplicaFailoverUtilsTest, TestSizeLimitedDistributedMap, TestConfigSetImmutable, DistributedSpellCheckComponentTest, FacetPivotSmallTest, TestXIncludeConfig, TestFieldCacheReopen, CoreMergeIndexesAdminHandlerTest, SolrCLIZkUtilsTest, PolyFieldTest, UnloadDistributedZkTest, TestXmlQParserPlugin, HdfsSyncSliceTest, OverseerRolesTest, ClassificationUpdateProcessorIntegrationTest, TestBinaryField, ChaosMonkeyNothingIsSafeTest, HLLSerializationTest, TestCharFilters, TestNonDefinedSimilarityFactory, DeleteLastCustomShardedReplicaTest, TestSolrDeletionPolicy1, BlockDirectoryTest, LeaderFailoverAfterPartitionTest, MultiTermTest, SuggestComponentContextFilterQueryTest, TestGraphTermsQParserPlugin, JSONWriterTest, TestExactSharedStatsCache, TestFieldSortValues, DateFieldTest, MoreLikeThisHandlerTest, PeerSyncTest, TestSchemaManager, OverseerCollectionConfigSetProcessorTest, SimpleFacetsTest, TestSort, DistributedFacetPivotLargeTest, TestRTGBase, SchemaVersionSpecificBehaviorTest, TestFunctionQuery, TestSegmentSorting, TestHdfsBackupRestoreCore, TestPKIAuthenticationPlugin, TestChildDocTransformer, TestDefaultStatsCache, TestSolrConfigHandler, DocValuesMultiTest, ExplicitHLLTest, SyncSliceTest, WrapperMergePolicyFactoryTest, TestObjectReleaseTracker, BooleanFieldTest, ShufflingReplicaListTransformerTest, BJQParserTest, TestNumericTerms64, PathHierarchyTokenizerFactoryTest, SolrCoreTest, SolrTestCaseJ4Test, TestDefaultSearchFieldResource, CdcrRequestHandlerTest, TestPivotHelperCode, AtomicUpdatesTest, TestConfigSets, TestSha256AuthenticationProvider, DirectSolrSpellCheckerTest, TestCloudRecovery, TestGraphMLResponseWriter, TestBulkSchemaConcurrent, TestSQLHandler, ClusterStateUpdateTest, JSONWriterTest, TestFastOutputStream, PreAnalyzedUpdateProcessorTest, CurrencyFieldXmlFileTest, HdfsBasicDistributedZkTest, TestJsonFacets, TestRestManager, DeleteReplicaTest, RecoveryZkTest, TestStressReorder, UniqFieldsUpdateProcessorFactoryTest, TestSimpleQParserPlugin, DateRangeFieldTest, TestReloadAndDeleteDocs, DOMUtilTest, TestDocTermOrds, TermsComponentTest, ZkSolrClientTest, ZkCLITest, ShardRoutingCustomTest, TestFaceting, TestHashPartitioner, ZkControllerTest, TestJoin, TestReload, TestStressVersions, HardAutoCommitTest, TestRangeQuery, TestCoreContainer, TestSolr4Spatial, StatsComponentTest, SpellCheckComponentTest, SolrCmdDistributorTest, QueryElevationComponentTest, BadIndexSchemaTest, ConvertedLegacyTest, TestFiltering, TestLazyCores, DirectUpdateHandlerTest, TestBadConfig, HighlighterTest, SoftAutoCommitTest, SolrIndexSplitterTest, SimplePostToolTest, TestCoreDiscovery, TestExtendedDismaxParser, SuggesterFSTTest, SolrRequestParserTest, DocValuesTest, TestTrie, SuggesterWFSTTest, TestCSVLoader, TestUpdate, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, QueryEqualityTest, StatelessScriptUpdateProcessorFactoryTest, SortByFunctionTest, DistanceFunctionTest, SolrInfoMBeanTest, LukeRequestHandlerTest, TestValueSourceCache, RequiredFieldsTest, IndexSchemaRuntimeFieldTest, IndexSchemaTest, UpdateRequestProcessorFactoryTest, QueryParsingTest, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, PingRequestHandlerTest, TestPostingsSolrHighlighter, TestPhraseSuggestions, SynonymTokenizerTest, TestSweetSpotSimilarityFactory, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestBM25SimilarityFactory, TestJmxMonitoredMap, PluginInfoTest, DateMathParserTest, TestSuggestSpellingConverter, TestSolrJ, ZkNodePropsTest, UUIDFieldTest, DistributedIntervalFacetingTest, ActionThrottleTest, AsyncCallRequestStatusResponseTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionReloadTest, CollectionsAPISolrJTest, DeleteShardTest, DistribDocExpirationUpdateProcessorTest, LeaderElectionContextKeyTest, LeaderInitiatedRecoveryOnCommitTest, LeaderInitiatedRecoveryOnShardRestartTest, MigrateRouteKeyTest, MultiThreadedOCPTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerStatusTest, PeerSyncReplicationTest, SSLMigrationTest, ShardSplitTest, TestRequestForwarding, TestRequestStatusCollectionAPI, TestStressCloudBlindAtomicUpdates, TestTolerantUpdateProcessorCloud, TestTolerantUpdateProcessorRandomCloud, VMParamsZkACLAndCredentialsProvidersTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsChaosMonkeySafeLeaderTest, HdfsCollectionsAPIDistributedZkTest, HdfsRecoveryZkTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=HdfsRecoveryZkTest -Dtests.seed=FC0068A0A00BD094 -Dtests.slow=true -Dtests.locale=sr-Latn -Dtests.timezone=America/North_Dakota/Beulah -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.00s J0 | HdfsRecoveryZkTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
[junit4] > at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
[junit4] > at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
[junit4] > at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
[junit4] > at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)
[junit4] > at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)
[junit4] > at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([FC0068A0A00BD094]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] Completed [529/662 (2!)] on J0 in 58.15s, 1 test, 1 failure <<< FAILURES!
[...truncated 62873 lines...]
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 560 -
Still unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/560/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC
1 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130) at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102) at sun.reflect.GeneratedConstructorAccessor102.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
at sun.reflect.GeneratedConstructorAccessor102.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)
at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)
at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([AB2AB2C2DA355053]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
at sun.reflect.GeneratedMethodAccessor30.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12393 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_AB2AB2C2DA355053-001/init-core-data-001
[junit4] 2> 2506117 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 2506118 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_AB2AB2C2DA355053-001/tempDir-001
[junit4] 2> 2506119 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2506119 INFO (Thread-8615) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2506119 INFO (Thread-8615) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2506219 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:37466
[junit4] 2> 2506233 INFO (jetty-launcher-9391-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2506233 INFO (jetty-launcher-9391-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2506234 INFO (jetty-launcher-9391-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4b942ed0{/solr,null,AVAILABLE}
[junit4] 2> 2506235 INFO (jetty-launcher-9391-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1f30322b{SSL,[ssl, http/1.1]}{127.0.0.1:54660}
[junit4] 2> 2506235 INFO (jetty-launcher-9391-thread-2) [ ] o.e.j.s.Server Started @2510470ms
[junit4] 2> 2506235 INFO (jetty-launcher-9391-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54660}
[junit4] 2> 2506235 ERROR (jetty-launcher-9391-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2506236 INFO (jetty-launcher-9391-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.4.0
[junit4] 2> 2506236 INFO (jetty-launcher-9391-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2506236 INFO (jetty-launcher-9391-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2506236 INFO (jetty-launcher-9391-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-19T15:03:21.258Z
[junit4] 2> 2506239 INFO (jetty-launcher-9391-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1c1c4ad6{/solr,null,AVAILABLE}
[junit4] 2> 2506240 INFO (jetty-launcher-9391-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3e318b62{SSL,[ssl, http/1.1]}{127.0.0.1:52909}
[junit4] 2> 2506240 INFO (jetty-launcher-9391-thread-1) [ ] o.e.j.s.Server Started @2510475ms
[junit4] 2> 2506240 INFO (jetty-launcher-9391-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52909}
[junit4] 2> 2506240 ERROR (jetty-launcher-9391-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2506240 INFO (jetty-launcher-9391-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.4.0
[junit4] 2> 2506241 INFO (jetty-launcher-9391-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2506241 INFO (jetty-launcher-9391-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2506241 INFO (jetty-launcher-9391-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-19T15:03:21.263Z
[junit4] 2> 2506247 INFO (jetty-launcher-9391-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2506247 INFO (jetty-launcher-9391-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2506258 INFO (jetty-launcher-9391-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2506259 INFO (jetty-launcher-9391-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2506259 INFO (jetty-launcher-9391-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37466/solr
[junit4] 2> 2506260 INFO (jetty-launcher-9391-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37466/solr
[junit4] 2> 2506311 INFO (jetty-launcher-9391-thread-1) [n:127.0.0.1:52909_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52909_solr
[junit4] 2> 2506313 INFO (jetty-launcher-9391-thread-1) [n:127.0.0.1:52909_solr ] o.a.s.c.Overseer Overseer (id=97134824727707654-127.0.0.1:52909_solr-n_0000000000) starting
[junit4] 2> 2506313 INFO (jetty-launcher-9391-thread-2) [n:127.0.0.1:54660_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54660_solr
[junit4] 2> 2506318 INFO (zkCallback-9402-thread-1-processing-n:127.0.0.1:52909_solr) [n:127.0.0.1:52909_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2506319 INFO (zkCallback-9401-thread-1-processing-n:127.0.0.1:54660_solr) [n:127.0.0.1:54660_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2506329 INFO (jetty-launcher-9391-thread-1) [n:127.0.0.1:52909_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52909_solr
[junit4] 2> 2506331 INFO (zkCallback-9402-thread-1-processing-n:127.0.0.1:52909_solr) [n:127.0.0.1:52909_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2506346 INFO (zkCallback-9401-thread-1-processing-n:127.0.0.1:54660_solr) [n:127.0.0.1:54660_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2506427 INFO (jetty-launcher-9391-thread-2) [n:127.0.0.1:54660_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_AB2AB2C2DA355053-001/tempDir-001/node2/.
[junit4] 2> 2506492 INFO (jetty-launcher-9391-thread-1) [n:127.0.0.1:52909_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_AB2AB2C2DA355053-001/tempDir-001/node1/.
[junit4] 2> 2506499 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x159179bc3370007, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2506530 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2506532 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37466/solr ready
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 2506586 WARN (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 2506598 WARN (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2506601 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2506622 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_solaris.vm_64907_hdfs____.9opxws/webapp
[junit4] 2> 2508280 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:64907
[junit4] 2> 2508477 WARN (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2508478 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2508500 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_63988_datanode____96xz43/webapp
[junit4] 2> 2509049 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:63988
[junit4] 2> 2509137 WARN (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2509138 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2509179 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_47622_datanode____666mm2/webapp
[junit4] 2> 2509221 INFO (IPC Server handler 3 on 35997) [ ] BlockStateChange BLOCK* processReport: from storage DS-2967eede-3e3a-4c6f-ac5c-b3422102455f node DatanodeRegistration(127.0.0.1:33027, datanodeUuid=dc4a6456-d7a6-4cf0-81cc-1817067525b1, infoPort=60374, infoSecurePort=0, ipcPort=40004, storageInfo=lv=-56;cid=testClusterID;nsid=935650325;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2509221 INFO (IPC Server handler 3 on 35997) [ ] BlockStateChange BLOCK* processReport: from storage DS-5af795bd-887f-46c1-8412-77369bd1662c node DatanodeRegistration(127.0.0.1:33027, datanodeUuid=dc4a6456-d7a6-4cf0-81cc-1817067525b1, infoPort=60374, infoSecurePort=0, ipcPort=40004, storageInfo=lv=-56;cid=testClusterID;nsid=935650325;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2512785 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:47622
[junit4] 2> 2512899 INFO (IPC Server handler 4 on 35997) [ ] BlockStateChange BLOCK* processReport: from storage DS-5c603127-a1b8-4349-904f-55a61182b128 node DatanodeRegistration(127.0.0.1:56611, datanodeUuid=6d6b8e2e-f3d5-4145-827a-5d3b047c59a1, infoPort=43606, infoSecurePort=0, ipcPort=64632, storageInfo=lv=-56;cid=testClusterID;nsid=935650325;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2512900 INFO (IPC Server handler 4 on 35997) [ ] BlockStateChange BLOCK* processReport: from storage DS-efffe18c-aef2-41e1-89f0-1abb2fa9ee43 node DatanodeRegistration(127.0.0.1:56611, datanodeUuid=6d6b8e2e-f3d5-4145-827a-5d3b047c59a1, infoPort=43606, infoSecurePort=0, ipcPort=64632, storageInfo=lv=-56;cid=testClusterID;nsid=935650325;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2512990 INFO (TEST-HdfsRecoveryZkTest.test-seed#[AB2AB2C2DA355053]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2513012 INFO (qtp1149546374-46278) [n:127.0.0.1:52909_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2513016 INFO (OverseerThreadFactory-9896-thread-1-processing-n:127.0.0.1:52909_solr) [n:127.0.0.1:52909_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 2513135 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 2513154 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 2513261 INFO (zkCallback-9402-thread-1-processing-n:127.0.0.1:52909_solr) [n:127.0.0.1:52909_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2513261 INFO (zkCallback-9401-thread-1-processing-n:127.0.0.1:54660_solr) [n:127.0.0.1:54660_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2514152 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 2514159 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema [recoverytest_shard1_replica2] Schema name=minimal
[junit4] 2> 2514161 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2514163 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica2' using configuration from collection recoverytest
[junit4] 2> 2514166 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 2514171 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:35997/data
[junit4] 2> 2514171 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 2514171 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [[recoverytest_shard1_replica2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_AB2AB2C2DA355053-001/tempDir-001/node1/recoverytest_shard1_replica2], dataDir=[hdfs://solaris-vm:35997/data/recoverytest/core_node1/data/]
[junit4] 2> 2514171 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:35997/data/recoverytest/core_node1/data/snapshot_metadata
[junit4] 2> 2514173 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema [recoverytest_shard1_replica1] Schema name=minimal
[junit4] 2> 2514176 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2514177 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica1' using configuration from collection recoverytest
[junit4] 2> 2514177 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:35997/data
[junit4] 2> 2514177 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 2514177 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [[recoverytest_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_AB2AB2C2DA355053-001/tempDir-001/node2/recoverytest_shard1_replica1], dataDir=[hdfs://solaris-vm:35997/data/recoverytest/core_node2/data/]
[junit4] 2> 2514178 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:35997/data/recoverytest/core_node2/data/snapshot_metadata
[junit4] 2> 2514180 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2514180 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2514180 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2514185 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2514185 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2514185 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2514293 WARN (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 2514295 WARN (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 2519294 WARN (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 2519295 WARN (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 2524301 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2524303 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2524304 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:35997/data/recoverytest/core_node1/data
[junit4] 2> 2524304 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:35997/data/recoverytest/core_node2/data
[junit4] 2> 2524325 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:35997/data/recoverytest/core_node1/data/index
[junit4] 2> 2524326 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:35997/data/recoverytest/core_node2/data/index
[junit4] 2> 2524333 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2524333 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2524333 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2524336 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2524337 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2524337 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2524425 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2524429 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2524534 INFO (IPC Server handler 2 on 35997) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:33027 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-efffe18c-aef2-41e1-89f0-1abb2fa9ee43:NORMAL:127.0.0.1:56611|RBW], ReplicaUC[[DISK]DS-5af795bd-887f-46c1-8412-77369bd1662c:NORMAL:127.0.0.1:33027|RBW]]} size 0
[junit4] 2> 2524535 INFO (IPC Server handler 9 on 35997) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:56611 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-5af795bd-887f-46c1-8412-77369bd1662c:NORMAL:127.0.0.1:33027|RBW], ReplicaUC[[DISK]DS-5c603127-a1b8-4349-904f-55a61182b128:NORMAL:127.0.0.1:56611|FINALIZED]]} size 0
[junit4] 2> 2524535 INFO (IPC Server handler 5 on 35997) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:56611 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-2967eede-3e3a-4c6f-ac5c-b3422102455f:NORMAL:127.0.0.1:33027|RBW], ReplicaUC[[DISK]DS-efffe18c-aef2-41e1-89f0-1abb2fa9ee43:NORMAL:127.0.0.1:56611|RBW]]} size 0
[junit4] 2> 2524537 INFO (IPC Server handler 0 on 35997) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:33027 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-2967eede-3e3a-4c6f-ac5c-b3422102455f:NORMAL:127.0.0.1:33027|RBW], ReplicaUC[[DISK]DS-efffe18c-aef2-41e1-89f0-1abb2fa9ee43:NORMAL:127.0.0.1:56611|RBW]]} size 0
[junit4] 2> 2524565 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 2524565 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2524565 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 2524570 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 2524570 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2524570 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 2524584 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2524584 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2524588 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2524588 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2524602 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4a8486a3[recoverytest_shard1_replica2] main]
[junit4] 2> 2524604 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2524605 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2524606 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2524606 INFO (searcherExecutor-9901-thread-1-processing-n:127.0.0.1:52909_solr x:recoverytest_shard1_replica2 s:shard1 c:recoverytest) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [recoverytest_shard1_replica2] Registered new searcher Searcher@4a8486a3[recoverytest_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2524606 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@66bfe60e[recoverytest_shard1_replica1] main]
[junit4] 2> 2524606 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1554157215026249728
[junit4] 2> 2524608 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2524608 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2524608 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2524609 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1554157215029395456
[junit4] 2> 2524609 INFO (searcherExecutor-9902-thread-1-processing-n:127.0.0.1:54660_solr x:recoverytest_shard1_replica1 s:shard1 c:recoverytest) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [recoverytest_shard1_replica1] Registered new searcher Searcher@66bfe60e[recoverytest_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2524620 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2524620 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2524620 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/
[junit4] 2> 2524621 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica2 url=https://127.0.0.1:52909/solr START replicas=[https://127.0.0.1:54660/solr/recoverytest_shard1_replica1/] nUpdates=100
[junit4] 2> 2524621 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica2 url=https://127.0.0.1:52909/solr DONE. We have no versions. sync failed.
[junit4] 2> 2524622 INFO (zkCallback-9402-thread-5-processing-n:127.0.0.1:52909_solr) [n:127.0.0.1:52909_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2524625 INFO (zkCallback-9401-thread-1-processing-n:127.0.0.1:54660_solr) [n:127.0.0.1:54660_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2524631 INFO (qtp1708883334-46264) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.S.Request [recoverytest_shard1_replica1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 2524632 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 2524632 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 2524642 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/ shard1
[junit4] 2> 2524749 INFO (zkCallback-9402-thread-4-processing-n:127.0.0.1:52909_solr) [n:127.0.0.1:52909_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2524749 INFO (zkCallback-9401-thread-1-processing-n:127.0.0.1:54660_solr) [n:127.0.0.1:54660_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2524794 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2524798 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=11669
[junit4] 2> 2524901 INFO (zkCallback-9402-thread-4-processing-n:127.0.0.1:52909_solr) [n:127.0.0.1:52909_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2524902 INFO (zkCallback-9401-thread-1-processing-n:127.0.0.1:54660_solr) [n:127.0.0.1:54660_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2525626 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=12479
[junit4] 2> 2525632 INFO (qtp1149546374-46278) [n:127.0.0.1:52909_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2525738 INFO (zkCallback-9402-thread-5-processing-n:127.0.0.1:52909_solr) [n:127.0.0.1:52909_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2525738 INFO (zkCallback-9401-thread-1-processing-n:127.0.0.1:54660_solr) [n:127.0.0.1:54660_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2526634 INFO (qtp1149546374-46278) [n:127.0.0.1:52909_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=13628
[junit4] 2> 2526635 INFO (TEST-HdfsRecoveryZkTest.test-seed#[AB2AB2C2DA355053]) [ ] o.a.s.c.RecoveryZkTest Indexing 1350 documents
[junit4] 2> 2526665 INFO (qtp1708883334-46266) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-0 (1554157217160101888)]} 0 15
[junit4] 2> 2526666 INFO (qtp1149546374-46274) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 (1554157217160101888)]} 0 25
[junit4] 2> 2526666 INFO (qtp1708883334-46260) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-0 (1554157217162199040)]} 0 15
[junit4] 2> 2526667 INFO (qtp1149546374-46283) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 (1554157217162199040)]} 0 23
[junit4] 2> 2526672 INFO (qtp1708883334-46264) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-1 (1554157217190510592)]} 0 0
[junit4] 2> 2526672 INFO (qtp1708883334-46263) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-1 (1554157217188413440)]} 0 1
[junit4] 2> 2526673 INFO (qtp1149546374-46278) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 (1554157217190510592)]} 0 3
[junit4] 2> 2526673 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 (1554157217188413440)]} 0 4
[junit4] 2> 2526678 INFO (qtp1708883334-46262) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-2 (1554157217195753473)]} 0 1
[junit4] 2> 2526678 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-2 (1554157217195753472)]} 0 1
[junit4] 2> 2526679 INFO (qtp1149546374-46285) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 (1554157217195753473)]} 0 3
[junit4] 2> 2526679 INFO (qtp1149546374-46276) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 (1554157217195753472)]} 0 3
[junit4] 2> 2526687 INFO (qtp1708883334-46259) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-3 (1554157217202044928)]} 0 2
[junit4] 2> 2526688 INFO (qtp1149546374-46274) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 (1554157217202044928)]} 0 6
[junit4] 2> 2526689 INFO (qtp1708883334-46266) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-3 (1554157217202044929)]} 0 1
[junit4] 2> 2526689 INFO (qtp1149546374-46275) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 (1554157217202044929)]} 0 8
[junit4] 2> 2526698 INFO (qtp1708883334-46264) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-2 (-1554157217214627840)]} 0 1
[junit4] 2> 2526699 INFO (qtp1708883334-46260) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-2 (-1554157217214627841)]} 0 2
[junit4] 2> 2526699 INFO (qtp1149546374-46278) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-2 (-1554157217214627841)]} 0 6
[junit4] 2> 2526700 INFO (qtp1149546374-46283) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-2 (-1554157217214627840)]} 0 6
[junit4] 2> 2526707 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-4 (1554157217225113600)]} 0 1
[junit4] 2> 2526707 INFO (qtp1708883334-46263) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-4 (1554157217224065024)]} 0 1
[junit4] 2> 2526707 INFO (qtp1149546374-46285) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 (1554157217225113600)]} 0 4
[junit4] 2> 2526708 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 (1554157217224065024)]} 0 5
[junit4] 2> 2526715 INFO (qtp1708883334-46266) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-5 (1554157217232453632)]} 0 1
[junit4] 2> 2526715 INFO (qtp1708883334-46259) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-5 (1554157217232453633)]} 0 1
[junit4] 2> 2526715 INFO (qtp1149546374-46274) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 (1554157217232453633)]} 0 5
[junit4] 2> 2526715 INFO (qtp1149546374-46276) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 (1554157217232453632)]} 0 5
[junit4] 2> 2526721 INFO (qtp1708883334-46264) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-6 (1554157217240842241)]} 0 1
[junit4] 2> 2526721 INFO (qtp1708883334-46262) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-6 (1554157217240842240)]} 0 1
[junit4] 2> 2526722 INFO (qtp1149546374-46275) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 (1554157217240842240)]} 0 3
[junit4] 2> 2526722 INFO (qtp1149546374-46278) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-6 (1554157217240842241)]} 0 3
[junit4] 2> 2526728 INFO (qtp1708883334-46260) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-7 (1554157217248182273)]} 0 1
[junit4] 2> 2526728 INFO (qtp1149546374-46285) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-7 (1554157217248182273)]} 0 3
[junit4] 2> 2526729 INFO (qtp1708883334-46263) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-7 (1554157217248182272)]} 0 1
[junit4] 2> 2526729 INFO (qtp1149546374-46283) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 (1554157217248182272)]} 0 4
[junit4] 2> 2526733 INFO (qtp1708883334-46259) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-4 (-1554157217253425152)]} 0 1
[junit4] 2> 2526734 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-4 (-1554157217253425152)]} 0 3
[junit4] 2> 2526734 INFO (qtp1708883334-46266) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-4 (-1554157217254473728)]} 0 1
[junit4] 2> 2526735 INFO (qtp1149546374-46274) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-4 (-1554157217254473728)]} 0 3
[junit4] 2> 2526743 INFO (qtp1708883334-46259) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-8 (1554157217260765184)]} 0 1
[junit4] 2> 2526743 INFO (qtp1708883334-46264) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-8 (1554157217260765185)]} 0 1
[junit4] 2> 2526743 INFO (qtp1149546374-46275) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 (1554157217260765185)]} 0 6
[junit4] 2> 2526743 INFO (qtp1149546374-46276) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-8 (1554157217260765184)]} 0 6
[junit4] 2> 2526749 INFO (qtp1708883334-46262) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-5 (-1554157217269153792)]} 0 1
[junit4] 2> 2526750 INFO (qtp1149546374-46278) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-5 (-1554157217269153792)]} 0 4
[junit4] 2> 2526751 INFO (qtp1708883334-46263) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-5 (-1554157217270202368)]} 0 2
[junit4] 2> 2526752 INFO (qtp1149546374-46285) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-5 (-1554157217270202368)]} 0 6
[junit4] 2> 2526755 INFO (qtp1708883334-46262) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-9 (1554157217276493824)]} 0 1
[junit4] 2> 2526755 INFO (qtp1149546374-46283) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 (1554157217276493824)]} 0 3
[junit4] 2> 2526758 INFO (qtp1708883334-46262) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-9 (1554157217278590976)]} 0 1
[junit4] 2> 2526758 INFO (qtp1149546374-46285) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-9 (1554157217278590976)]} 0 3
[junit4] 2> 2526760 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-6 (-1554157217281736704)]} 0 0
[junit4] 2> 2526760 INFO (qtp1149546374-46283) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-6 (-1554157217281736704)]} 0 2
[junit4] 2> 2526764 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-6 (-1554157217285931008)]} 0 1
[junit4] 2> 2526764 INFO (qtp1149546374-46285) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-6 (-1554157217285931008)]} 0 3
[junit4] 2> 2526767 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-10 (1554157217288028160)]} 0 1
[junit4] 2> 2526769 INFO (qtp1149546374-46274) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 (1554157217288028160)]} 0 5
[junit4] 2> 2526769 INFO (qtp1708883334-46264) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-10 (1554157217291173888)]} 0 1
[junit4] 2> 2526770 INFO (qtp1149546374-46285) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-10 (1554157217291173888)]} 0 3
[junit4] 2> 2526773 INFO (qtp1708883334-46264) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-8 (-1554157217296416768)]} 0 1
[junit4] 2> 2526774 INFO (qtp1149546374-46278) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-8 (-1554157217296416768)]} 0 3
[junit4] 2> 2526778 INFO (qtp1708883334-46262) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-8 (-1554157217297465344)]} 0 2
[junit4] 2> 2526778 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-8 (-1554157217297465344)]} 0 6
[junit4] 2> 2526780 INFO (qtp1708883334-46266) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-11 (1554157217302708224)]} 0 1
[junit4] 2> 2526780 INFO (qtp1149546374-46283) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-11 (1554157217302708224)]} 0 3
[junit4] 2> 2526784 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-11 (1554157217306902528)]} 0 1
[junit4] 2> 2526784 INFO (qtp1149546374-46275) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-11 (1554157217306902528)]} 0 3
[junit4] 2> 2526787 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-9 (-1554157217308999680)]} 0 1
[junit4] 2> 2526787 INFO (qtp1149546374-46276) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-9 (-1554157217308999680)]} 0 3
[junit4] 2> 2526790 INFO (qtp1708883334-46260) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-9 (-1554157217312145408)]} 0 1
[junit4] 2> 2526790 INFO (qtp1149546374-46274) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-9 (-1554157217312145408)]} 0 3
[junit4] 2> 2526792 INFO (qtp1708883334-46264) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-12 (1554157217316339712)]} 0 1
[junit4] 2> 2526793 INFO (qtp1149546374-46285) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-12 (1554157217316339712)]} 0 3
[junit4] 2> 2526795 INFO (qtp1708883334-46259) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-12 (1554157217318436864)]} 0 0
[junit4] 2> 2526795 INFO (qtp1149546374-46278) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-12 (1554157217318436864)]} 0 3
[junit4] 2> 2526798 INFO (qtp1708883334-46262) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-13 (1554157217321582592)]} 0 1
[junit4] 2> 2526799 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-13 (1554157217321582592)]} 0 4
[junit4] 2> 2526800 INFO (qtp1708883334-46266) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-13 (1554157217323679744)]} 0 1
[junit4] 2> 2526801 INFO (qtp1149546374-46283) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-13 (1554157217323679744)]} 0 3
[junit4] 2> 2526805 INFO (qtp1708883334-46262) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-14 (1554157217328922624)]} 0 1
[junit4] 2> 2526805 INFO (qtp1149546374-46275) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-14 (1554157217328922624)]} 0 3
[junit4] 2> 2526807 INFO (qtp1708883334-46265) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-14 (1554157217331019776)]} 0 1
[junit4] 2> 2526807 INFO (qtp1149546374-46276) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-14 (1554157217331019776)]} 0 3
[junit4] 2> 2526810 INFO (qtp1708883334-46260) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-10 (-1554157217334165504)]} 0 1
[junit4] 2> 2526811 INFO (qtp1149546374-46274) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-10 (-1554157217334165504)]} 0 3
[junit4] 2> 2526812 INFO (qtp1708883334-46264) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-10 (-1554157217336262656)]} 0 0
[junit4] 2> 2526812 INFO (qtp1149546374-46285) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-10 (-1554157217336262656)]} 0 2
[junit4] 2> 2526817 INFO (qtp1708883334-46264) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-15 (1554157217341505536)]} 0 1
[junit4] 2> 2526818 INFO (qtp1149546374-46278) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-15 (1554157217341505536)]} 0 4
[junit4] 2> 2526819 INFO (qtp1708883334-46263) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-15 (1554157217341505537)]} 0 1
[junit4] 2> 2526819 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-15 (1554157217341505537)]} 0 4
[junit4] 2> 2526823 INFO (qtp1708883334-46266) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-13 (-1554157217347796992)]} 0 0
[junit4] 2> 2526823 INFO (qtp1149546374-46283) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-13 (-1554157217347796992)]} 0 2
[junit4] 2> 2526824 INFO (qtp1708883334-46262) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-13 (-1554157217348845568)]} 0 0
[junit4] 2> 2526825 INFO (qtp1149546374-46275) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-13 (-1554157217348845568)]} 0 3
[junit4] 2> 2526829 INFO (qtp1708883334-46266) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-16 (1554157217353039872)]} 0 1
[junit4] 2> 2526829 INFO (qtp1149546374-46276) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-16 (1554157217353039872)]} 0 4
[junit4] 2> 2526830 INFO (qtp1708883334-46260) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-16 (1554157217355137024)]} 0 1
[junit4] 2> 2526830 INFO (qtp1149546374-46274) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-16 (1554157217355137024)]} 0 3
[junit4] 2> 2526834 INFO (qtp1708883334-46259) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-17 (1554157217360379904)]} 0 0
[junit4] 2> 2526835 INFO (qtp1149546374-46285) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-17 (1554157217360379904)]} 0 3
[junit4] 2> 2526837 INFO (qtp1708883334-46264) [n:127.0.0.1:54660_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52909/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-17 (1554157217361428480)]} 0 1
[junit4] 2> 2526838 INFO (qtp1149546374-46278) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-17 (1554157217361428480)]} 0 5
[junit4] 2> 2526842 ERROR (updateExecutor-9397-thread-1-processing-https:////127.0.0.1:54660//solr//recoverytest_shard1_replica1 x:recoverytest_shard1_replica2 r:core_node1 n:127.0.0.1:52909_solr s:shard1 c:recoverytest) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.StreamingSolrClients error
[junit4] 2> org.apache.http.NoHttpResponseException: 127.0.0.1:54660 failed to respond
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
[junit4] 2> at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
[junit4] 2> at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2526844 WARN (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.DistributedUpdateProcessor Error sending update to https://127.0.0.1:54660/solr
[junit4] 2> org.apache.http.NoHttpResponseException: 127.0.0.1:54660 failed to respond
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
[junit4] 2> at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
[junit4] 2> at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2526844 ERROR (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on replica https://127.0.0.1:54660/solr/recoverytest_shard1_replica1/
[junit4] 2> org.apache.http.NoHttpResponseException: 127.0.0.1:54660 failed to respond
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
[junit4] 2> at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
[junit4] 2> at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2526844 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.ZkController Put replica core=recoverytest_shard1_replica1 coreNodeName=core_node2 on 127.0.0.1:54660_solr into leader-initiated recovery.
[junit4] 2> 2526845 INFO (qtp1149546374-46284) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-18 (1554157217365622784)]} 0 7
[junit4] 2> 2526845 INFO (TEST-HdfsRecoveryZkTest.test-seed#[AB2AB2C2DA355053]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@1f30322b{SSL,[ssl, http/1.1]}{127.0.0.1:0}
[junit4] 2> 2526852 INFO (updateExecutor-9397-thread-1-processing-x:recoverytest_shard1_replica2 r:core_node1 https:////127.0.0.1:54660//solr//recoverytest_shard1_replica1// n:127.0.0.1:52909_solr s:shard1 c:recoverytest) [n:127.0.0.1:52909_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.LeaderInitiatedRecoveryThread Put replica core=recoverytest_shard1_replica1 coreNodeName=core_node2 on 127.0.0.1:54660_solr into leader-initiated recovery.
[junit4] 2> 2526852 WARN (updateExecutor-9397-thread-1-proc
[...truncated too long message...]
java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> 2539809 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37466 37466
[junit4] 2> 2539810 INFO (SUITE-HdfsRecoveryZkTest-seed#[AB2AB2C2DA355053]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {rnd_b=Lucene50(blocksize=128), _version_=PostingsFormat(name=LuceneVarGapDocFreqInterval), a_t=PostingsFormat(name=LuceneVarGapDocFreqInterval), a_i=Lucene50(blocksize=128), id=Lucene50(blocksize=128)}, docValues:{}, maxPointsInLeafNode=2008, maxMBSortInHeap=7.968954124163064, sim=RandomSimilarity(queryNorm=true,coord=no): {}, locale=es-CO, timezone=Africa/Windhoek
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=2,free=223475544,total=536870912
[junit4] 2> NOTE: All tests run in this JVM: [LeaderInitiatedRecoveryOnShardRestartTest, DefaultValueUpdateProcessorTest, TestIndexingPerformance, CollectionTooManyReplicasTest, MBeansHandlerTest, TestSchemaSimilarityResource, SolrIndexSplitterTest, TestFieldCacheSortRandom, TestSSLRandomization, TestHdfsUpdateLog, ExitableDirectoryReaderTest, TestCustomSort, SimpleMLTQParserTest, TestSolrCloudWithDelegationTokens, TestFilteredDocIdSet, CoreSorterTest, EchoParamsTest, MinimalSchemaTest, HighlighterMaxOffsetTest, TestIndexSearcher, DistributedIntervalFacetingTest, TestShortCircuitedRequests, TestStressUserVersions, WordBreakSolrSpellCheckerTest, HdfsThreadLeakTest, SearchHandlerTest, FileBasedSpellCheckerTest, TestDocBasedVersionConstraints, TestSmileRequest, TestSearchPerf, RequestHandlersTest, SolrCoreCheckLockOnStartupTest, TestSolrQueryParser, ZkSolrClientTest, ConvertedLegacyTest, TestSolrDynamicMBean, SmileWriterTest, NotRequiredUniqueKeyTest, TestSchemaNameResource, TestNRTOpen, URLClassifyProcessorTest, TestQueryWrapperFilter, TestManagedStopFilterFactory, TestSolrCloudSnapshots, BinaryUpdateRequestHandlerTest, HdfsRestartWhileUpdatingTest, TestManagedSchemaThreadSafety, ClassificationUpdateProcessorTest, TestAuthenticationFramework, TestTolerantUpdateProcessorRandomCloud, RulesTest, TermVectorComponentDistributedTest, TestMinMaxOnMultiValuedField, TestExactStatsCache, TestLMJelinekMercerSimilarityFactory, SolrCloudExampleTest, TestSolrDeletionPolicy2, TimerUtilsTest, BadCopyFieldTest, ShardRoutingCustomTest, TestLMDirichletSimilarityFactory, TestSuggestSpellingConverter, TestFieldCacheWithThreads, SortByFunctionTest, SimpleCollectionCreateDeleteTest, SpellingQueryConverterTest, BufferStoreTest, CdcrReplicationHandlerTest, SpellPossibilityIteratorTest, ConnectionReuseTest, AnalyticsMergeStrategyTest, TestDistributedGrouping, FieldMutatingUpdateProcessorTest, CursorMarkTest, TestCloudSchemaless, TestCrossCoreJoin, OverriddenZkACLAndCredentialsProvidersTest, LukeRequestHandlerTest, TestSubQueryTransformerDistrib, TestJsonFacetRefinement, TestValueSourceCache, TestConfigOverlay, TestInfoStreamLogging, TestPerFieldSimilarityClassic, AddSchemaFieldsUpdateProcessorFactoryTest, CollectionsAPISolrJTest, TestFuzzyAnalyzedSuggestions, LeaderElectionIntegrationTest, TestSolrCloudWithSecureImpersonation, TestDocSet, CacheHeaderTest, QueryResultKeyTest, TolerantUpdateProcessorTest, TestCloudInspectUtil, TestFiltering, ChangedSchemaMergeTest, HLLUtilTest, DocValuesMissingTest, HdfsBasicDistributedZk2Test, TestShardHandlerFactory, MultiThreadedOCPTest, HdfsChaosMonkeyNothingIsSafeTest, SegmentsInfoRequestHandlerTest, RollingRestartTest, TestConfigReload, TestSerializedLuceneMatchVersion, SpellCheckComponentTest, TestUseDocValuesAsStored, TestRecoveryHdfs, RemoteQueryErrorTest, DistributedFacetPivotSmallTest, LeaderElectionContextKeyTest, AsyncCallRequestStatusResponseTest, StressHdfsTest, BigEndianAscendingWordDeserializerTest, TestRangeQuery, NoCacheHeaderTest, TestHdfsCloudBackupRestore, TestTrie, TestDeleteCollectionOnDownNodes, AutoCommitTest, TemplateUpdateProcessorTest, ConfigSetsAPITest, TestSolrConfigHandlerConcurrent, BlockJoinFacetSimpleTest, TestMaxScoreQueryParser, RAMDirectoryFactoryTest, TestRestoreCore, TestMiniSolrCloudClusterSSL, DeleteShardTest, TestDFISimilarityFactory, TestOmitPositions, FullSolrCloudDistribCmdsTest, TestLocalFSCloudBackupRestore, OverseerModifyCollectionTest, ShowFileRequestHandlerTest, DistributedFacetExistsSmallTest, ClassificationUpdateProcessorFactoryTest, ConjunctionSolrSpellCheckerTest, ResourceLoaderTest, TestCloudManagedSchema, TestLuceneMatchVersion, TestDistribIDF, TestScoreJoinQPNoScore, DistributedQueryComponentCustomSortTest, BitVectorTest, AnalysisErrorHandlingTest, CollectionsAPIDistributedZkTest, ParsingFieldUpdateProcessorsTest, SmileWriterTest, DistributedDebugComponentTest, ExternalFileFieldSortTest, JsonLoaderTest, SpellCheckCollatorTest, TestRTimerTree, CreateCollectionCleanupTest, FieldAnalysisRequestHandlerTest, TestDistributedMissingSort, PreAnalyzedFieldTest, CoreAdminRequestStatusTest, DistributedTermsComponentTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, OpenCloseCoreStressTest, ShardRoutingTest, TestRandomFaceting, TestZkChroot, TestRandomDVFaceting, ZkCLITest, TestDistributedSearch, IndexSchemaTest, TestCollationField, UpdateRequestProcessorFactoryTest, JSONWriterTest, QueryParsingTest, UniqFieldsUpdateProcessorFactoryTest, TestPHPSerializedResponseWriter, TestLFUCache, TestSimpleTrackingShardHandler, AssignTest, CdcrBootstrapTest, CollectionsAPIAsyncDistributedZkTest, DistribCursorPagingTest, DistribJoinFromCollectionTest, DistributedVersionInfoTest, LeaderFailureAfterFreshStartTest, LeaderInitiatedRecoveryOnCommitTest, OverseerRolesTest, OverseerStatusTest, OverseerTaskQueueTest, RecoveryAfterSoftCommitTest, ReplicationFactorTest, SSLMigrationTest, SaslZkACLProviderTest, SharedFSAutoReplicaFailoverTest, SharedFSAutoReplicaFailoverUtilsTest, TestCloudPivotFacet, TestCloudRecovery, TestConfigSetsAPIZkFailure, TestDownShardTolerantSearch, TestLeaderElectionZkExpiry, TestLeaderInitiatedRecoveryThread, TestOnReconnectListenerSupport, TestRebalanceLeaders, TestSegmentSorting, TestSolrCloudWithKerberosAlt, TestStressLiveNodes, VMParamsZkACLAndCredentialsProvidersTest, HdfsBasicDistributedZkTest, HdfsCollectionsAPIDistributedZkTest, HdfsNNFailoverTest, HdfsRecoveryZkTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=HdfsRecoveryZkTest -Dtests.seed=AB2AB2C2DA355053 -Dtests.slow=true -Dtests.locale=es-CO -Dtests.timezone=Africa/Windhoek -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.00s J1 | HdfsRecoveryZkTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
[junit4] > at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
[junit4] > at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
[junit4] > at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
[junit4] > at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)
[junit4] > at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)
[junit4] > at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([AB2AB2C2DA355053]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] Completed [540/662 (1!)] on J1 in 64.79s, 1 test, 1 failure <<< FAILURES!
[...truncated 62805 lines...]
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 559 -
Failure!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/559/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
3 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.core.TestLazyCores
Error Message:
ObjectTracker found 5 object(s) that were not released!!! [MDCAwareThreadPoolExecutor, MockDirectoryWrapper, SolrCore, MockDirectoryWrapper, MockDirectoryWrapper] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:797) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347) at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:331) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:640) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:846) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:936) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:66) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:672) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:846) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347) at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:433) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:840) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 5 object(s) that were not released!!! [MDCAwareThreadPoolExecutor, MockDirectoryWrapper, SolrCore, MockDirectoryWrapper, MockDirectoryWrapper]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:797)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:331)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:640)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:846)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:936)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:66)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:672)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:846)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:433)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:840)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([520991A99683960]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
at sun.reflect.GeneratedMethodAccessor35.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
FAILED: junit.framework.TestSuite.org.apache.solr.core.TestLazyCores
Error Message:
1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 1) Thread[id=46846, name=searcherExecutor-9169-thread-1, state=WAITING, group=TGRP-TestLazyCores] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:
1) Thread[id=46846, name=searcherExecutor-9169-thread-1, state=WAITING, group=TGRP-TestLazyCores]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([520991A99683960]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.core.TestLazyCores
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=46846, name=searcherExecutor-9169-thread-1, state=WAITING, group=TGRP-TestLazyCores] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=46846, name=searcherExecutor-9169-thread-1, state=WAITING, group=TGRP-TestLazyCores]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([520991A99683960]:0)
Build Log:
[...truncated 12081 lines...]
[junit4] Suite: org.apache.solr.core.TestLazyCores
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/init-core-data-001
[junit4] 2> 1508712 INFO (SUITE-TestLazyCores-seed#[520991A99683960]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 1508714 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testLazySearch
[junit4] 2> 1508734 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
[junit4] 2> 1508734 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ ] o.a.s.c.SolrCores Allocating transient cache for 4 transient cores
[junit4] 2> 1508746 INFO (coreLoadExecutor-8949-thread-3) [ x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1508746 INFO (coreLoadExecutor-8949-thread-2) [ x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1508746 INFO (coreLoadExecutor-8949-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1508758 INFO (coreLoadExecutor-8949-thread-3) [ x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
[junit4] 2> 1508758 INFO (coreLoadExecutor-8949-thread-2) [ x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
[junit4] 2> 1508758 INFO (coreLoadExecutor-8949-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
[junit4] 2> 1508761 INFO (coreLoadExecutor-8949-thread-3) [ x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1508761 INFO (coreLoadExecutor-8949-thread-2) [ x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1508761 INFO (coreLoadExecutor-8949-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1508763 INFO (coreLoadExecutor-8949-thread-3) [ x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection5
[junit4] 2> 1508763 INFO (coreLoadExecutor-8949-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection1
[junit4] 2> 1508763 INFO (coreLoadExecutor-8949-thread-2) [ x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection2
[junit4] 2> 1508763 INFO (coreLoadExecutor-8949-thread-3) [ x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection5/data/]
[junit4] 2> 1508763 INFO (coreLoadExecutor-8949-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection1/data/]
[junit4] 2> 1508763 INFO (coreLoadExecutor-8949-thread-2) [ x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection2/data/]
[junit4] 2> 1508814 INFO (coreLoadExecutor-8949-thread-3) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=88.0234375, floorSegmentMB=1.5947265625, forceMergeDeletesPctAllowed=0.9103532200750553, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1732849212417416
[junit4] 2> 1508815 INFO (coreLoadExecutor-8949-thread-2) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=88.0234375, floorSegmentMB=1.5947265625, forceMergeDeletesPctAllowed=0.9103532200750553, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1732849212417416
[junit4] 2> 1508815 INFO (coreLoadExecutor-8949-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=88.0234375, floorSegmentMB=1.5947265625, forceMergeDeletesPctAllowed=0.9103532200750553, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1732849212417416
[junit4] 2> 1508824 INFO (coreLoadExecutor-8949-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1508824 INFO (coreLoadExecutor-8949-thread-3) [ x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1508825 INFO (coreLoadExecutor-8949-thread-2) [ x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1508825 INFO (coreLoadExecutor-8949-thread-2) [ x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1508825 INFO (coreLoadExecutor-8949-thread-3) [ x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1508825 INFO (coreLoadExecutor-8949-thread-2) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5213739801295223]
[junit4] 2> 1508825 INFO (coreLoadExecutor-8949-thread-3) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5213739801295223]
[junit4] 2> 1508824 INFO (coreLoadExecutor-8949-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1508825 INFO (coreLoadExecutor-8949-thread-2) [ x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4688bf3b[collection2] main]
[junit4] 2> 1508825 INFO (coreLoadExecutor-8949-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5213739801295223]
[junit4] 2> 1508826 INFO (coreLoadExecutor-8949-thread-2) [ x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection2/conf
[junit4] 2> 1508826 INFO (coreLoadExecutor-8949-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4f31d93c[collection1] main]
[junit4] 2> 1508825 INFO (coreLoadExecutor-8949-thread-3) [ x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@27ea89a1[collection5] main]
[junit4] 2> 1508826 INFO (coreLoadExecutor-8949-thread-2) [ x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1508826 INFO (coreLoadExecutor-8949-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection1/conf
[junit4] 2> 1508826 INFO (coreLoadExecutor-8949-thread-3) [ x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection5/conf
[junit4] 2> 1508826 INFO (coreLoadExecutor-8949-thread-2) [ x:collection2] o.a.s.c.SolrCores Opening transient core collection2
[junit4] 2> 1508826 INFO (coreLoadExecutor-8949-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1508826 INFO (coreLoadExecutor-8949-thread-3) [ x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1508826 INFO (searcherExecutor-8951-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4f31d93c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1508826 INFO (searcherExecutor-8952-thread-1-processing-x:collection2) [ x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@4688bf3b[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1508826 INFO (searcherExecutor-8950-thread-1-processing-x:collection5) [ x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@27ea89a1[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1508833 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1508839 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema [collection4] Schema name=tiny
[junit4] 2> 1508840 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1508842 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'collection4' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection4
[junit4] 2> 1508842 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCore [[collection4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection4/data/]
[junit4] 2> 1508843 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=34, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=53.8115234375, floorSegmentMB=1.8408203125, forceMergeDeletesPctAllowed=2.164381828279139, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1508850 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1508850 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1508850 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=54.3701171875, floorSegmentMB=2.0126953125, forceMergeDeletesPctAllowed=0.3762357144658579, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1508850 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@2148ebdd[collection4] main]
[junit4] 2> 1508850 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-001/collection4/conf
[junit4] 2> 1508850 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1508851 INFO (searcherExecutor-8962-thread-1-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [collection4] Registered new searcher Searcher@2148ebdd[collection4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1508851 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1508851 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@47ff90a3
[junit4] 2> 1508852 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@6a4d9088[collection4] main]
[junit4] 2> 1508853 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1508853 INFO (searcherExecutor-8962-thread-1-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [collection4] Registered new searcher Searcher@6a4d9088[collection4] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.4.0):C7)))}
[junit4] 2> 1508853 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.c.S.Request [collection4] webapp=null path=null params={q={!prefix+f%3Dv_t}hel&wt=xml} hits=2 status=0 QTime=0
[junit4] 2> 1508855 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.c.S.Request [collection4] webapp=null path=null params={q={!raw+f%3Dv_t}hello&wt=xml} hits=2 status=0 QTime=0
[junit4] 2> 1508855 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.c.S.Request [collection4] webapp=null path=null params={q={!raw+f%3Dv_t}Hello&wt=xml} hits=0 status=0 QTime=0
[junit4] 2> 1508855 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.c.S.Request [collection4] webapp=null path=null params={q={!raw+f%3Dv_f}1.5&wt=xml} hits=0 status=0 QTime=0
[junit4] 2> 1508856 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.c.S.Request [collection1] webapp=null path=null params={q={!raw+f%3Dv_t}hello&wt=xml} hits=0 status=0 QTime=0
[junit4] 2> 1508856 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.c.CoreContainer Shutting down CoreContainer instance=349540913
[junit4] 2> 1508856 INFO (coreCloseExecutor-8966-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@34f6f628
[junit4] 2> 1508857 INFO (coreCloseExecutor-8966-thread-2-processing-x:collection4) [ x:collection5] o.a.s.c.SolrCore [collection5] CLOSING SolrCore org.apache.solr.core.SolrCore@75950045
[junit4] 2> 1508857 INFO (coreCloseExecutor-8966-thread-3-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [collection4] CLOSING SolrCore org.apache.solr.core.SolrCore@15f2c6d7
[junit4] 2> 1508858 INFO (coreCloseExecutor-8966-thread-4-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCore [collection2] CLOSING SolrCore org.apache.solr.core.SolrCore@48204fbd
[junit4] 2> 1508859 INFO (TEST-TestLazyCores.testLazySearch-seed#[520991A99683960]) [ x:collection4] o.a.s.SolrTestCaseJ4 ###Ending testLazySearch
[junit4] 2> 1508860 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.SolrTestCaseJ4 ###Starting testCreateSame
[junit4] 2> 1508879 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
[junit4] 2> 1508879 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCores Allocating transient cache for 4 transient cores
[junit4] 2> 1508896 INFO (coreLoadExecutor-8973-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1508896 INFO (coreLoadExecutor-8973-thread-3-processing-x:collection4) [ x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1508896 INFO (coreLoadExecutor-8973-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1508902 INFO (coreLoadExecutor-8973-thread-2-processing-x:collection4) [ x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
[junit4] 2> 1508902 INFO (coreLoadExecutor-8973-thread-1-processing-x:collection4) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
[junit4] 2> 1508902 INFO (coreLoadExecutor-8973-thread-3-processing-x:collection4) [ x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
[junit4] 2> 1508905 INFO (coreLoadExecutor-8973-thread-2-processing-x:collection4) [ x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1508905 INFO (coreLoadExecutor-8973-thread-3-processing-x:collection4) [ x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1508905 INFO (coreLoadExecutor-8973-thread-1-processing-x:collection4) [ x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1508907 INFO (coreLoadExecutor-8973-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection2
[junit4] 2> 1508907 INFO (coreLoadExecutor-8973-thread-3-processing-x:collection4) [ x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection5
[junit4] 2> 1508907 INFO (coreLoadExecutor-8973-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection1
[junit4] 2> 1508907 INFO (coreLoadExecutor-8973-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection2/data/]
[junit4] 2> 1508907 INFO (coreLoadExecutor-8973-thread-3-processing-x:collection4) [ x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection5/data/]
[junit4] 2> 1508907 INFO (coreLoadExecutor-8973-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection1/data/]
[junit4] 2> 1508908 INFO (coreLoadExecutor-8973-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=88.0234375, floorSegmentMB=1.5947265625, forceMergeDeletesPctAllowed=0.9103532200750553, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1732849212417416
[junit4] 2> 1508908 INFO (coreLoadExecutor-8973-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=88.0234375, floorSegmentMB=1.5947265625, forceMergeDeletesPctAllowed=0.9103532200750553, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1732849212417416
[junit4] 2> 1508909 INFO (coreLoadExecutor-8973-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=88.0234375, floorSegmentMB=1.5947265625, forceMergeDeletesPctAllowed=0.9103532200750553, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1732849212417416
[junit4] 2> 1508917 INFO (coreLoadExecutor-8973-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1508917 INFO (coreLoadExecutor-8973-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1508917 INFO (coreLoadExecutor-8973-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5213739801295223]
[junit4] 2> 1508917 INFO (coreLoadExecutor-8973-thread-3-processing-x:collection4) [ x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@5d4bcd4f[collection5] main]
[junit4] 2> 1508917 INFO (coreLoadExecutor-8973-thread-3-processing-x:collection4) [ x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection5/conf
[junit4] 2> 1508917 INFO (coreLoadExecutor-8973-thread-3-processing-x:collection4) [ x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1508918 INFO (searcherExecutor-8975-thread-1-processing-x:collection5) [ x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@5d4bcd4f[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1508924 INFO (coreLoadExecutor-8973-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1508924 INFO (coreLoadExecutor-8973-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1508925 INFO (coreLoadExecutor-8973-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1508924 INFO (coreLoadExecutor-8973-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1508925 INFO (coreLoadExecutor-8973-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5213739801295223]
[junit4] 2> 1508925 INFO (coreLoadExecutor-8973-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5213739801295223]
[junit4] 2> 1508925 INFO (coreLoadExecutor-8973-thread-2-processing-x:collection4) [ x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@56d6380[collection2] main]
[junit4] 2> 1508925 INFO (coreLoadExecutor-8973-thread-1-processing-x:collection4) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3ac25556[collection1] main]
[junit4] 2> 1508925 INFO (coreLoadExecutor-8973-thread-2-processing-x:collection4) [ x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection2/conf
[junit4] 2> 1508925 INFO (coreLoadExecutor-8973-thread-1-processing-x:collection4) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection1/conf
[junit4] 2> 1508925 INFO (coreLoadExecutor-8973-thread-2-processing-x:collection4) [ x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1508925 INFO (coreLoadExecutor-8973-thread-1-processing-x:collection4) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1508925 INFO (coreLoadExecutor-8973-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCores Opening transient core collection2
[junit4] 2> 1508925 INFO (searcherExecutor-8976-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3ac25556[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1508926 INFO (searcherExecutor-8974-thread-1-processing-x:collection2) [ x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@56d6380[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1508936 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1508941 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema [collection4] Schema name=tiny
[junit4] 2> 1508943 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1508944 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'collection4' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection4
[junit4] 2> 1508945 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCore [[collection4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection4/data/]
[junit4] 2> 1508946 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=33, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3226232710072548]
[junit4] 2> 1508952 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1508952 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1508952 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 1508953 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@2efcbd15[collection4] main]
[junit4] 2> 1508953 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection4/conf
[junit4] 2> 1508953 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1508953 INFO (searcherExecutor-8987-thread-1-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [collection4] Registered new searcher Searcher@2efcbd15[collection4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1508959 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1508965 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema [collection6] Schema name=tiny
[junit4] 2> 1508967 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1508968 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'collection6' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection6
[junit4] 2> 1508968 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCore [[collection6] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection6], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection6/data/]
[junit4] 2> 1508969 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5865174074779143]
[junit4] 2> 1508980 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1508980 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1508980 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=13, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 1508980 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@73d55e5b[collection6] main]
[junit4] 2> 1508981 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-002/collection6/conf
[junit4] 2> 1508981 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1508981 INFO (searcherExecutor-8991-thread-1-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [collection6] Registered new searcher Searcher@73d55e5b[collection6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1508982 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCores Opening transient core collection6
[junit4] 2> 1508995 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.h.a.CoreAdminOperation core create command schema=schema.xml&dataDir=t12&name=collection2&action=CREATE&config=solrconfig.xml&wt=xml
[junit4] 2> 1508995 WARN (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.c.CoreContainer Creating a core with existing name is not allowed
[junit4] 2> 1508995 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.h.a.CoreAdminOperation core create command schema=schema.xml&dataDir=t14&name=collection4&action=CREATE&config=solrconfig.xml&wt=xml
[junit4] 2> 1508996 WARN (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.c.CoreContainer Creating a core with existing name is not allowed
[junit4] 2> 1508996 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.h.a.CoreAdminOperation core create command schema=schema.xml&dataDir=t15&name=collection5&action=CREATE&config=solrconfig.xml&wt=xml
[junit4] 2> 1508996 WARN (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.c.CoreContainer Creating a core with existing name is not allowed
[junit4] 2> 1508996 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.h.a.CoreAdminOperation core create command schema=schema.xml&dataDir=t16&name=collection6&action=CREATE&config=solrconfig.xml&wt=xml
[junit4] 2> 1508996 WARN (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.c.CoreContainer Creating a core with existing name is not allowed
[junit4] 2> 1508997 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1240344310
[junit4] 2> 1508997 INFO (coreCloseExecutor-8995-thread-1-processing-x:collection4) [ x:collection5] o.a.s.c.SolrCore [collection5] CLOSING SolrCore org.apache.solr.core.SolrCore@393f33b0
[junit4] 2> 1508997 INFO (coreCloseExecutor-8995-thread-4-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCore [collection2] CLOSING SolrCore org.apache.solr.core.SolrCore@3b2b9976
[junit4] 2> 1508997 INFO (coreCloseExecutor-8995-thread-2-processing-x:collection4) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@472997a7
[junit4] 2> 1508998 INFO (coreCloseExecutor-8995-thread-3-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [collection4] CLOSING SolrCore org.apache.solr.core.SolrCore@69d96cf5
[junit4] 2> 1508998 INFO (coreCloseExecutor-8995-thread-5-processing-x:collection4) [ x:collection6] o.a.s.c.SolrCore [collection6] CLOSING SolrCore org.apache.solr.core.SolrCore@40f3a103
[junit4] 2> 1509000 INFO (TEST-TestLazyCores.testCreateSame-seed#[520991A99683960]) [ x:collection4] o.a.s.SolrTestCaseJ4 ###Ending testCreateSame
[junit4] 2> 1509002 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.SolrTestCaseJ4 ###Starting testCreateTransientFromAdmin
[junit4] 2> 1509022 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
[junit4] 2> 1509023 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCores Allocating transient cache for 4 transient cores
[junit4] 2> 1509089 INFO (coreLoadExecutor-9002-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1509094 INFO (coreLoadExecutor-9002-thread-2-processing-x:collection4) [ x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
[junit4] 2> 1509095 INFO (coreLoadExecutor-9002-thread-3-processing-x:collection4) [ x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1509096 INFO (coreLoadExecutor-9002-thread-2-processing-x:collection4) [ x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1509098 INFO (coreLoadExecutor-9002-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/collection2
[junit4] 2> 1509099 INFO (coreLoadExecutor-9002-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/collection2/data/]
[junit4] 2> 1509099 INFO (coreLoadExecutor-9002-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=88.0234375, floorSegmentMB=1.5947265625, forceMergeDeletesPctAllowed=0.9103532200750553, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1732849212417416
[junit4] 2> 1509101 INFO (coreLoadExecutor-9002-thread-3-processing-x:collection4) [ x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
[junit4] 2> 1509103 INFO (coreLoadExecutor-9002-thread-3-processing-x:collection4) [ x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1509104 INFO (coreLoadExecutor-9002-thread-3-processing-x:collection4) [ x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/collection5
[junit4] 2> 1509104 INFO (coreLoadExecutor-9002-thread-3-processing-x:collection4) [ x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/collection5/data/]
[junit4] 2> 1509105 INFO (coreLoadExecutor-9002-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=88.0234375, floorSegmentMB=1.5947265625, forceMergeDeletesPctAllowed=0.9103532200750553, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1732849212417416
[junit4] 2> 1509108 INFO (coreLoadExecutor-9002-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1509108 INFO (coreLoadExecutor-9002-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1509108 INFO (coreLoadExecutor-9002-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5213739801295223]
[junit4] 2> 1509108 INFO (coreLoadExecutor-9002-thread-2-processing-x:collection4) [ x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@779226b8[collection2] main]
[junit4] 2> 1509108 INFO (coreLoadExecutor-9002-thread-2-processing-x:collection4) [ x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/collection2/conf
[junit4] 2> 1509109 INFO (coreLoadExecutor-9002-thread-2-processing-x:collection4) [ x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1509109 INFO (coreLoadExecutor-9002-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCores Opening transient core collection2
[junit4] 2> 1509109 INFO (searcherExecutor-9003-thread-1-processing-x:collection2) [ x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@779226b8[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1509112 INFO (coreLoadExecutor-9002-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1509112 INFO (coreLoadExecutor-9002-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1509112 INFO (coreLoadExecutor-9002-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5213739801295223]
[junit4] 2> 1509113 INFO (coreLoadExecutor-9002-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1509118 INFO (coreLoadExecutor-9002-thread-1-processing-x:collection4) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
[junit4] 2> 1509124 INFO (coreLoadExecutor-9002-thread-3-processing-x:collection4) [ x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@d781b1f[collection5] main]
[junit4] 2> 1509125 INFO (coreLoadExecutor-9002-thread-3-processing-x:collection4) [ x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/collection5/conf
[junit4] 2> 1509129 INFO (coreLoadExecutor-9002-thread-3-processing-x:collection4) [ x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1509129 INFO (searcherExecutor-9004-thread-1-processing-x:collection5) [ x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@d781b1f[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1509184 INFO (coreLoadExecutor-9002-thread-1-processing-x:collection4) [ x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1509186 INFO (coreLoadExecutor-9002-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/collection1
[junit4] 2> 1509186 INFO (coreLoadExecutor-9002-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/collection1/data/]
[junit4] 2> 1509187 INFO (coreLoadExecutor-9002-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=88.0234375, floorSegmentMB=1.5947265625, forceMergeDeletesPctAllowed=0.9103532200750553, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1732849212417416
[junit4] 2> 1509198 INFO (coreLoadExecutor-9002-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1509198 INFO (coreLoadExecutor-9002-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1509198 INFO (coreLoadExecutor-9002-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5213739801295223]
[junit4] 2> 1509199 INFO (coreLoadExecutor-9002-thread-1-processing-x:collection4) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3ae7bc25[collection1] main]
[junit4] 2> 1509199 INFO (coreLoadExecutor-9002-thread-1-processing-x:collection4) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/collection1/conf
[junit4] 2> 1509200 INFO (coreLoadExecutor-9002-thread-1-processing-x:collection4) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1509200 INFO (searcherExecutor-9011-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3ae7bc25[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1509211 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.h.a.CoreAdminOperation core create command loadOnStartup=true&transient=true&name=core1&action=CREATE&wt=xml
[junit4] 2> 1509223 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1509229 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema [core1] Schema name=tiny
[junit4] 2> 1509231 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1509232 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'core1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core1
[junit4] 2> 1509233 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCore [[core1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core1/data/]
[junit4] 2> 1509234 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=24, maxMergedSegmentMB=29.8125, floorSegmentMB=1.1689453125, forceMergeDeletesPctAllowed=27.13113335889392, segmentsPerTier=21.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1509241 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1509241 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1509241 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=12, maxMergedSegmentMB=98.85546875, floorSegmentMB=0.60546875, forceMergeDeletesPctAllowed=4.535142330063099, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 1509241 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@7c13da3f[core1] main]
[junit4] 2> 1509241 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core1/conf
[junit4] 2> 1509242 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1509242 INFO (searcherExecutor-9016-thread-1-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [core1] Registered new searcher Searcher@7c13da3f[core1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1509242 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCores Opening transient core core1
[junit4] 2> 1509242 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.h.a.CoreAdminOperation core create command loadOnStartup=false&transient=true&name=core2&action=CREATE&wt=xml
[junit4] 2> 1509248 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1509254 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema [core2] Schema name=tiny
[junit4] 2> 1509256 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1509258 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'core2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core2
[junit4] 2> 1509258 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCore [[core2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core2/data/]
[junit4] 2> 1509259 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1460580337, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 1509266 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1509266 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1509266 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.21686142414330442]
[junit4] 2> 1509267 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@5d8a67[core2] main]
[junit4] 2> 1509267 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core2/conf
[junit4] 2> 1509267 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1509267 INFO (searcherExecutor-9021-thread-1-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [core2] Registered new searcher Searcher@5d8a67[core2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1509267 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCores Opening transient core core2
[junit4] 2> 1509267 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.h.a.CoreAdminOperation core create command loadOnStartup=true&transient=true&name=core3&action=CREATE&wt=xml
[junit4] 2> 1509273 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1509287 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema [core3] Schema name=tiny
[junit4] 2> 1509290 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1509292 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'core3' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core3
[junit4] 2> 1509293 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCore [[core3] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core3], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core3/data/]
[junit4] 2> 1509294 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=25.6396484375, floorSegmentMB=0.302734375, forceMergeDeletesPctAllowed=16.42753129537579, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1509305 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1509305 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1509305 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=25, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3568148663910117]
[junit4] 2> 1509306 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@4e7bc8d0[core3] main]
[junit4] 2> 1509306 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core3/conf
[junit4] 2> 1509307 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1509307 INFO (searcherExecutor-9026-thread-1-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [core3] Registered new searcher Searcher@4e7bc8d0[core3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1509307 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCores Opening transient core core3
[junit4] 2> 1509307 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.h.a.CoreAdminOperation core create command loadOnStartup=false&transient=true&name=core4&action=CREATE&wt=xml
[junit4] 2> 1509319 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1509327 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema [core4] Schema name=tiny
[junit4] 2> 1509329 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1509331 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'core4' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core4
[junit4] 2> 1509332 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCore [[core4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core4/data/]
[junit4] 2> 1509333 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=41, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7253951871115877]
[junit4] 2> 1509348 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1509348 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1509348 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=59.7763671875, floorSegmentMB=1.630859375, forceMergeDeletesPctAllowed=22.379556021780754, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.13203128033445194
[junit4] 2> 1509349 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@1928d10b[core4] main]
[junit4] 2> 1509349 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core4/conf
[junit4] 2> 1509349 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1509350 INFO (searcherExecutor-9031-thread-1-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [core4] Registered new searcher Searcher@1928d10b[core4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1509350 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCores Opening transient core core4
[junit4] 2> 1509350 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCores Closing transient core [collection2]
[junit4] 2> 1509350 INFO (Thread-6837) [ x:collection4] o.a.s.c.SolrCore [collection2] CLOSING SolrCore org.apache.solr.core.SolrCore@38525fc7
[junit4] 2> 1509350 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.h.a.CoreAdminOperation core create command loadOnStartup=false&transient=true&name=core5&action=CREATE&wt=xml
[junit4] 2> 1509361 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1509370 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema [core5] Schema name=tiny
[junit4] 2> 1509373 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1509376 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'core5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core5
[junit4] 2> 1509376 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCore [[core5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core5/data/]
[junit4] 2> 1509378 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=34, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=37.60546875, floorSegmentMB=0.333984375, forceMergeDeletesPctAllowed=11.04321729253638, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.33448656571925417
[junit4] 2> 1509389 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1509389 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1509389 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=10, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 1509390 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@5d60379b[core5] main]
[junit4] 2> 1509390 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core5/conf
[junit4] 2> 1509390 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1509390 INFO (searcherExecutor-9036-thread-1-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [core5] Registered new searcher Searcher@5d60379b[core5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1509391 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCores Opening transient core core5
[junit4] 2> 1509391 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCores Closing transient core [core1]
[junit4] 2> 1509391 INFO (Thread-6837) [ x:collection4] o.a.s.c.SolrCore [core1] CLOSING SolrCore org.apache.solr.core.SolrCore@7516541c
[junit4] 2> 1509465 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1509473 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema [core1] Schema name=tiny
[junit4] 2> 1509476 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 1509477 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'core1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core1
[junit4] 2> 1509478 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.c.SolrCore [[core1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_520991A99683960-001/tempDir-003/core1/data/]
[junit4] 2> 1509479 INFO (TEST-TestLazyCores.testCreateTransientFromAdmin-seed#[520991A99683960]) [ x:collection4] o.a.s.u.RandomMerge
[...truncated too long message...]
eTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
[junit4] > at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:331)
[junit4] > at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:640)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:846)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:936)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
[junit4] > at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:66)
[junit4] > at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:672)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:846)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:347)
[junit4] > at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:433)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:840)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([520991A99683960]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
[junit4] > at java.lang.Thread.run(Thread.java:745)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:
[junit4] > 1) Thread[id=46846, name=searcherExecutor-9169-thread-1, state=WAITING, group=TGRP-TestLazyCores]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([520991A99683960]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=46846, name=searcherExecutor-9169-thread-1, state=WAITING, group=TGRP-TestLazyCores]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([520991A99683960]:0)
[junit4] Completed [457/662 (1!)] on J1 in 179.64s, 8 tests, 1 failure, 2 errors <<< FAILURES!
[...truncated 44974 lines...]
-documentation-lint:
[echo] checking for broken html...
[jtidy] Checking for broken html (such as invalid tags)...
[delete] Deleting directory /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/jtidy_tmp
[echo] Checking for broken links...
[exec]
[exec] Crawl/parse...
[exec]
[exec] Verify...
[exec]
[exec] file:///build/docs/misc/org/apache/lucene/search/DocValuesStats.DoubleDocValuesStats.html
[exec] BROKEN LINK: file:///build/docs/core/org/apache/lucene/search/DocValuesStats.NumericDocValuesStats.html
[exec]
[exec] file:///build/docs/misc/org/apache/lucene/search/DocValuesStats.LongDocValuesStats.html
[exec] BROKEN LINK: file:///build/docs/core/org/apache/lucene/search/DocValuesStats.NumericDocValuesStats.html
[exec]
[exec] file:///build/docs/misc/org/apache/lucene/search/DocValuesStats.SortedDoubleDocValuesStats.html
[exec] BROKEN LINK: file:///build/docs/core/org/apache/lucene/search/DocValuesStats.SortedNumericDocValuesStats.html
[exec]
[exec] file:///build/docs/misc/org/apache/lucene/search/DocValuesStats.SortedLongDocValuesStats.html
[exec] BROKEN LINK: file:///build/docs/core/org/apache/lucene/search/DocValuesStats.SortedNumericDocValuesStats.html
[exec]
[exec] Broken javadocs links were found!
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:765: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:101: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build.xml:138: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build.xml:151: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:2478: exec returned: 1
Total time: 62 minutes 10 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 558 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/558/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC
1 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130) at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102) at sun.reflect.GeneratedConstructorAccessor160.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
at sun.reflect.GeneratedConstructorAccessor160.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)
at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)
at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([53EFA0D1A06992C1]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12335 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_53EFA0D1A06992C1-001/init-core-data-001
[junit4] 2> 2305287 INFO (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 2305287 INFO (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_53EFA0D1A06992C1-001/tempDir-001
[junit4] 2> 2305288 INFO (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2305288 INFO (Thread-7992) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2305288 INFO (Thread-7992) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2305388 INFO (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:35620
[junit4] 2> 2305396 INFO (jetty-launcher-10305-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2305396 INFO (jetty-launcher-10305-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2305399 INFO (jetty-launcher-10305-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@70117e37{/solr,null,AVAILABLE}
[junit4] 2> 2305399 INFO (jetty-launcher-10305-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@965c7b0{/solr,null,AVAILABLE}
[junit4] 2> 2305400 INFO (jetty-launcher-10305-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6a134fd8{HTTP/1.1,[http/1.1]}{127.0.0.1:56154}
[junit4] 2> 2305401 INFO (jetty-launcher-10305-thread-2) [ ] o.e.j.s.Server Started @2309387ms
[junit4] 2> 2305401 INFO (jetty-launcher-10305-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5cff0f85{HTTP/1.1,[http/1.1]}{127.0.0.1:40691}
[junit4] 2> 2305401 INFO (jetty-launcher-10305-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=56154}
[junit4] 2> 2305401 INFO (jetty-launcher-10305-thread-1) [ ] o.e.j.s.Server Started @2309388ms
[junit4] 2> 2305401 INFO (jetty-launcher-10305-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40691}
[junit4] 2> 2305401 ERROR (jetty-launcher-10305-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2305401 ERROR (jetty-launcher-10305-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2305401 INFO (jetty-launcher-10305-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 2305401 INFO (jetty-launcher-10305-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 2305401 INFO (jetty-launcher-10305-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2305401 INFO (jetty-launcher-10305-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2305401 INFO (jetty-launcher-10305-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2305402 INFO (jetty-launcher-10305-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2305402 INFO (jetty-launcher-10305-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-16T14:17:43.181Z
[junit4] 2> 2305402 INFO (jetty-launcher-10305-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-16T14:17:43.182Z
[junit4] 2> 2305405 INFO (jetty-launcher-10305-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2305405 INFO (jetty-launcher-10305-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2305412 INFO (jetty-launcher-10305-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2305412 INFO (jetty-launcher-10305-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2305412 INFO (jetty-launcher-10305-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35620/solr
[junit4] 2> 2305413 INFO (jetty-launcher-10305-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35620/solr
[junit4] 2> 2305450 INFO (jetty-launcher-10305-thread-2) [n:127.0.0.1:56154_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56154_solr
[junit4] 2> 2305451 INFO (jetty-launcher-10305-thread-1) [n:127.0.0.1:40691_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40691_solr
[junit4] 2> 2305452 INFO (jetty-launcher-10305-thread-2) [n:127.0.0.1:56154_solr ] o.a.s.c.Overseer Overseer (id=97117658354089990-127.0.0.1:56154_solr-n_0000000000) starting
[junit4] 2> 2305454 INFO (zkCallback-10315-thread-1-processing-n:127.0.0.1:40691_solr) [n:127.0.0.1:40691_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2305454 INFO (zkCallback-10316-thread-1-processing-n:127.0.0.1:56154_solr) [n:127.0.0.1:56154_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2305467 INFO (jetty-launcher-10305-thread-2) [n:127.0.0.1:56154_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56154_solr
[junit4] 2> 2305469 INFO (zkCallback-10315-thread-1-processing-n:127.0.0.1:40691_solr) [n:127.0.0.1:40691_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2305469 INFO (zkCallback-10316-thread-1-processing-n:127.0.0.1:56154_solr) [n:127.0.0.1:56154_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2305515 INFO (jetty-launcher-10305-thread-2) [n:127.0.0.1:56154_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_53EFA0D1A06992C1-001/tempDir-001/node2/.
[junit4] 2> 2305551 INFO (jetty-launcher-10305-thread-1) [n:127.0.0.1:40691_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_53EFA0D1A06992C1-001/tempDir-001/node1/.
[junit4] 2> 2305558 INFO (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2305560 INFO (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35620/solr ready
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 2305896 WARN (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 2305904 WARN (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2305906 INFO (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2305932 INFO (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_solaris.vm_51368_hdfs____.odrvit/webapp
[junit4] 2> 2306644 INFO (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:51368
[junit4] 2> 2306756 WARN (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2306759 INFO (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2306779 INFO (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_41326_datanode____xnn0n5/webapp
[junit4] 2> 2307263 INFO (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41326
[junit4] 2> 2307350 WARN (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2307351 INFO (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2307365 INFO (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_44855_datanode____.tazao1/webapp
[junit4] 2> 2307576 INFO (IPC Server handler 3 on 60872) [ ] BlockStateChange BLOCK* processReport: from storage DS-27e48ad2-452e-4fbb-b1fd-d18485d808ad node DatanodeRegistration(127.0.0.1:34207, datanodeUuid=a1d90a12-a7c6-4786-93a0-f363cee03bb6, infoPort=39867, infoSecurePort=0, ipcPort=36111, storageInfo=lv=-56;cid=testClusterID;nsid=155682651;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2307577 INFO (IPC Server handler 3 on 60872) [ ] BlockStateChange BLOCK* processReport: from storage DS-c77cf730-5f50-445a-bd68-89dbddb4d260 node DatanodeRegistration(127.0.0.1:34207, datanodeUuid=a1d90a12-a7c6-4786-93a0-f363cee03bb6, infoPort=39867, infoSecurePort=0, ipcPort=36111, storageInfo=lv=-56;cid=testClusterID;nsid=155682651;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2307881 INFO (SUITE-HdfsRecoveryZkTest-seed#[53EFA0D1A06992C1]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44855
[junit4] 2> 2307995 INFO (IPC Server handler 6 on 60872) [ ] BlockStateChange BLOCK* processReport: from storage DS-bf638c2a-296f-4f44-8150-9a4113a48a37 node DatanodeRegistration(127.0.0.1:48098, datanodeUuid=7ee11719-2bd4-4949-b75e-b5e79d1644bf, infoPort=40792, infoSecurePort=0, ipcPort=62649, storageInfo=lv=-56;cid=testClusterID;nsid=155682651;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2307996 INFO (IPC Server handler 6 on 60872) [ ] BlockStateChange BLOCK* processReport: from storage DS-d95a6da6-c4d0-4992-b3b5-6bac85eb2f9c node DatanodeRegistration(127.0.0.1:48098, datanodeUuid=7ee11719-2bd4-4949-b75e-b5e79d1644bf, infoPort=40792, infoSecurePort=0, ipcPort=62649, storageInfo=lv=-56;cid=testClusterID;nsid=155682651;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2308075 INFO (TEST-HdfsRecoveryZkTest.test-seed#[53EFA0D1A06992C1]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2308079 INFO (qtp913267724-48512) [n:127.0.0.1:40691_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2308083 INFO (OverseerThreadFactory-12015-thread-1-processing-n:127.0.0.1:56154_solr) [n:127.0.0.1:56154_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 2308196 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 2308196 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 2308306 INFO (zkCallback-10315-thread-1-processing-n:127.0.0.1:40691_solr) [n:127.0.0.1:40691_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2308306 INFO (zkCallback-10316-thread-4-processing-n:127.0.0.1:56154_solr) [n:127.0.0.1:56154_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2309209 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 2309212 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 2309217 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema [recoverytest_shard1_replica1] Schema name=minimal
[junit4] 2> 2309218 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema [recoverytest_shard1_replica2] Schema name=minimal
[junit4] 2> 2309220 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2309220 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2309221 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica1' using configuration from collection recoverytest
[junit4] 2> 2309221 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica2' using configuration from collection recoverytest
[junit4] 2> 2309228 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:60872/data
[junit4] 2> 2309228 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 2309229 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [[recoverytest_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_53EFA0D1A06992C1-001/tempDir-001/node2/recoverytest_shard1_replica1], dataDir=[hdfs://solaris-vm:60872/data/recoverytest/core_node1/data/]
[junit4] 2> 2309229 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:60872/data
[junit4] 2> 2309229 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:60872/data/recoverytest/core_node1/data/snapshot_metadata
[junit4] 2> 2309229 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 2309229 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [[recoverytest_shard1_replica2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_53EFA0D1A06992C1-001/tempDir-001/node1/recoverytest_shard1_replica2], dataDir=[hdfs://solaris-vm:60872/data/recoverytest/core_node2/data/]
[junit4] 2> 2309229 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:60872/data/recoverytest/core_node2/data/snapshot_metadata
[junit4] 2> 2309237 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2309237 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2309237 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2309237 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2309237 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2309237 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2309244 WARN (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 2309244 WARN (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 2314273 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2314273 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2314274 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:60872/data/recoverytest/core_node2/data
[junit4] 2> 2314274 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:60872/data/recoverytest/core_node1/data
[junit4] 2> 2314298 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:60872/data/recoverytest/core_node2/data/index
[junit4] 2> 2314303 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:60872/data/recoverytest/core_node1/data/index
[junit4] 2> 2314307 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2314307 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2314307 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2314312 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2314314 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2314315 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2314315 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2314327 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2314357 INFO (IPC Server handler 2 on 60872) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:48098 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-27e48ad2-452e-4fbb-b1fd-d18485d808ad:NORMAL:127.0.0.1:34207|RBW], ReplicaUC[[DISK]DS-bf638c2a-296f-4f44-8150-9a4113a48a37:NORMAL:127.0.0.1:48098|RBW]]} size 0
[junit4] 2> 2314357 INFO (IPC Server handler 7 on 60872) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34207 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-bf638c2a-296f-4f44-8150-9a4113a48a37:NORMAL:127.0.0.1:48098|RBW], ReplicaUC[[DISK]DS-c77cf730-5f50-445a-bd68-89dbddb4d260:NORMAL:127.0.0.1:34207|FINALIZED]]} size 0
[junit4] 2> 2314362 INFO (IPC Server handler 0 on 60872) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34207 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-27e48ad2-452e-4fbb-b1fd-d18485d808ad:NORMAL:127.0.0.1:34207|RBW], ReplicaUC[[DISK]DS-d95a6da6-c4d0-4992-b3b5-6bac85eb2f9c:NORMAL:127.0.0.1:48098|RBW]]} size 0
[junit4] 2> 2314362 INFO (IPC Server handler 0 on 60872) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:48098 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-27e48ad2-452e-4fbb-b1fd-d18485d808ad:NORMAL:127.0.0.1:34207|RBW], ReplicaUC[[DISK]DS-d95a6da6-c4d0-4992-b3b5-6bac85eb2f9c:NORMAL:127.0.0.1:48098|RBW]]} size 0
[junit4] 2> 2314373 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 2314373 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2314373 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 2314384 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2314384 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2314398 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 2314398 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2314398 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 2314406 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@1b2128ec[recoverytest_shard1_replica2] main]
[junit4] 2> 2314408 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2314408 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2314408 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2314409 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2314409 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2314409 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1553882543228452864
[junit4] 2> 2314412 INFO (searcherExecutor-12021-thread-1-processing-n:127.0.0.1:40691_solr x:recoverytest_shard1_replica2 s:shard1 c:recoverytest) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [recoverytest_shard1_replica2] Registered new searcher Searcher@1b2128ec[recoverytest_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2314417 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@141234c[recoverytest_shard1_replica1] main]
[junit4] 2> 2314419 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2314419 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2314419 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2314420 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1553882543239987200
[junit4] 2> 2314422 INFO (searcherExecutor-12020-thread-1-processing-n:127.0.0.1:56154_solr x:recoverytest_shard1_replica1 s:shard1 c:recoverytest) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [recoverytest_shard1_replica1] Registered new searcher Searcher@141234c[recoverytest_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2314429 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2314429 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2314430 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/
[junit4] 2> 2314430 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica1 url=http://127.0.0.1:56154/solr START replicas=[http://127.0.0.1:40691/solr/recoverytest_shard1_replica2/] nUpdates=100
[junit4] 2> 2314431 INFO (zkCallback-10316-thread-3-processing-n:127.0.0.1:56154_solr) [n:127.0.0.1:56154_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2314431 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica1 url=http://127.0.0.1:56154/solr DONE. We have no versions. sync failed.
[junit4] 2> 2314432 INFO (zkCallback-10315-thread-1-processing-n:127.0.0.1:40691_solr) [n:127.0.0.1:40691_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2314436 INFO (qtp913267724-48519) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.c.S.Request [recoverytest_shard1_replica2] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 2314436 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 2314436 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 2314442 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/ shard1
[junit4] 2> 2314551 INFO (zkCallback-10316-thread-4-processing-n:127.0.0.1:56154_solr) [n:127.0.0.1:56154_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2314552 INFO (zkCallback-10315-thread-1-processing-n:127.0.0.1:40691_solr) [n:127.0.0.1:40691_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2314600 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2314609 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=6416
[junit4] 2> 2314721 INFO (zkCallback-10316-thread-3-processing-n:127.0.0.1:56154_solr) [n:127.0.0.1:56154_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2314721 INFO (zkCallback-10315-thread-1-processing-n:127.0.0.1:40691_solr) [n:127.0.0.1:40691_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2315433 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=7242
[junit4] 2> 2315437 INFO (qtp913267724-48512) [n:127.0.0.1:40691_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2315541 INFO (zkCallback-10315-thread-1-processing-n:127.0.0.1:40691_solr) [n:127.0.0.1:40691_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2315541 INFO (zkCallback-10316-thread-1-processing-n:127.0.0.1:56154_solr) [n:127.0.0.1:56154_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2316090 INFO (OverseerCollectionConfigSetProcessor-97117658354089990-127.0.0.1:56154_solr-n_0000000000) [n:127.0.0.1:56154_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2316438 INFO (qtp913267724-48512) [n:127.0.0.1:40691_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=8363
[junit4] 2> 2316439 INFO (TEST-HdfsRecoveryZkTest.test-seed#[53EFA0D1A06992C1]) [ ] o.a.s.c.RecoveryZkTest Indexing 700 documents
[junit4] 2> 2316472 INFO (qtp913267724-48520) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-0 (1553882545362305024)]} 0 18
[junit4] 2> 2316472 INFO (qtp1254808657-48513) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 (1553882545362305024)]} 0 28
[junit4] 2> 2316473 INFO (qtp913267724-48507) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-0 (1553882545362305025)]} 0 19
[junit4] 2> 2316473 INFO (qtp1254808657-48514) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 (1553882545362305025)]} 0 29
[junit4] 2> 2316478 INFO (qtp913267724-48516) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-1 (1553882545393762304)]} 0 1
[junit4] 2> 2316478 INFO (qtp1254808657-48515) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 (1553882545393762304)]} 0 3
[junit4] 2> 2316479 INFO (qtp913267724-48518) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-1 (1553882545394810880)]} 0 1
[junit4] 2> 2316479 INFO (qtp1254808657-48521) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 (1553882545394810880)]} 0 3
[junit4] 2> 2316483 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-0 (-1553882545401102336)]} 0 0
[junit4] 2> 2316483 INFO (qtp913267724-48519) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-0 (-1553882545400053760)]} 0 1
[junit4] 2> 2316483 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-0 (-1553882545401102336)]} 0 2
[junit4] 2> 2316483 INFO (qtp1254808657-48523) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-0 (-1553882545400053760)]} 0 3
[junit4] 2> 2316488 INFO (qtp913267724-48520) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-2 (1553882545405296641)]} 0 0
[junit4] 2> 2316488 INFO (qtp913267724-48512) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-2 (1553882545405296640)]} 0 1
[junit4] 2> 2316488 INFO (qtp1254808657-48513) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 (1553882545405296641)]} 0 2
[junit4] 2> 2316488 INFO (qtp1254808657-48509) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 (1553882545405296640)]} 0 3
[junit4] 2> 2316492 INFO (qtp913267724-48516) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-3 (1553882545410539521)]} 0 0
[junit4] 2> 2316492 INFO (qtp913267724-48507) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-3 (1553882545410539520)]} 0 0
[junit4] 2> 2316492 INFO (qtp1254808657-48515) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 (1553882545410539521)]} 0 2
[junit4] 2> 2316493 INFO (qtp1254808657-48514) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 (1553882545410539520)]} 0 2
[junit4] 2> 2316497 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-4 (1553882545415782400)]} 0 0
[junit4] 2> 2316497 INFO (qtp913267724-48518) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-4 (1553882545414733824)]} 0 1
[junit4] 2> 2316497 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 (1553882545415782400)]} 0 2
[junit4] 2> 2316498 INFO (qtp1254808657-48521) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 (1553882545414733824)]} 0 3
[junit4] 2> 2316502 INFO (qtp913267724-48520) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-1 (-1553882545421025280)]} 0 0
[junit4] 2> 2316502 INFO (qtp913267724-48519) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-1 (-1553882545419976704)]} 0 1
[junit4] 2> 2316502 INFO (qtp1254808657-48513) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-1 (-1553882545421025280)]} 0 2
[junit4] 2> 2316502 INFO (qtp1254808657-48523) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-1 (-1553882545419976704)]} 0 3
[junit4] 2> 2316509 INFO (qtp913267724-48516) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-5 (1553882545428365313)]} 0 0
[junit4] 2> 2316509 INFO (qtp913267724-48512) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-5 (1553882545428365312)]} 0 1
[junit4] 2> 2316510 INFO (qtp1254808657-48515) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 (1553882545428365313)]} 0 2
[junit4] 2> 2316510 INFO (qtp1254808657-48509) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 (1553882545428365312)]} 0 2
[junit4] 2> 2316514 INFO (qtp913267724-48507) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-6 (1553882545432559616)]} 0 1
[junit4] 2> 2316514 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-6 (1553882545432559617)]} 0 1
[junit4] 2> 2316515 INFO (qtp1254808657-48514) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-6 (1553882545432559616)]} 0 3
[junit4] 2> 2316515 INFO (qtp1254808657-48521) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 (1553882545432559617)]} 0 3
[junit4] 2> 2316519 INFO (qtp913267724-48520) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-7 (1553882545438851072)]} 0 0
[junit4] 2> 2316519 INFO (qtp913267724-48518) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-7 (1553882545437802496)]} 0 1
[junit4] 2> 2316519 INFO (qtp1254808657-48523) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 (1553882545438851072)]} 0 2
[junit4] 2> 2316520 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-7 (1553882545437802496)]} 0 3
[junit4] 2> 2316524 INFO (qtp913267724-48516) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-8 (1553882545443045376)]} 0 0
[junit4] 2> 2316524 INFO (qtp913267724-48519) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-8 (1553882545443045377)]} 0 1
[junit4] 2> 2316524 INFO (qtp1254808657-48513) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-8 (1553882545443045376)]} 0 2
[junit4] 2> 2316524 INFO (qtp1254808657-48515) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 (1553882545443045377)]} 0 3
[junit4] 2> 2316528 INFO (qtp913267724-48507) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-3 (-1553882545448288257)]} 0 0
[junit4] 2> 2316528 INFO (qtp913267724-48512) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-3 (-1553882545448288256)]} 0 0
[junit4] 2> 2316528 INFO (qtp1254808657-48514) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-3 (-1553882545448288257)]} 0 2
[junit4] 2> 2316528 INFO (qtp1254808657-48509) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-3 (-1553882545448288256)]} 0 2
[junit4] 2> 2316533 INFO (qtp913267724-48520) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-9 (1553882545452482561)]} 0 0
[junit4] 2> 2316533 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-9 (1553882545452482560)]} 0 1
[junit4] 2> 2316533 INFO (qtp1254808657-48521) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 (1553882545452482560)]} 0 3
[junit4] 2> 2316533 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-9 (1553882545452482561)]} 0 3
[junit4] 2> 2316538 INFO (qtp913267724-48516) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-10 (1553882545457725441)]} 0 0
[junit4] 2> 2316538 INFO (qtp913267724-48518) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-10 (1553882545457725440)]} 0 1
[junit4] 2> 2316538 INFO (qtp1254808657-48513) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-10 (1553882545457725441)]} 0 2
[junit4] 2> 2316538 INFO (qtp1254808657-48523) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 (1553882545457725440)]} 0 3
[junit4] 2> 2316542 INFO (qtp913267724-48519) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-5 (-1553882545462968321)]} 0 0
[junit4] 2> 2316542 INFO (qtp913267724-48507) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-5 (-1553882545462968320)]} 0 0
[junit4] 2> 2316543 INFO (qtp1254808657-48514) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-5 (-1553882545462968321)]} 0 2
[junit4] 2> 2316543 INFO (qtp1254808657-48515) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-5 (-1553882545462968320)]} 0 2
[junit4] 2> 2316547 INFO (qtp913267724-48520) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-11 (1553882545468211200)]} 0 0
[junit4] 2> 2316547 INFO (qtp913267724-48512) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-11 (1553882545467162624)]} 0 1
[junit4] 2> 2316547 INFO (qtp1254808657-48521) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-11 (1553882545468211200)]} 0 2
[junit4] 2> 2316547 INFO (qtp1254808657-48509) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-11 (1553882545467162624)]} 0 3
[junit4] 2> 2316552 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-12 (1553882545472405504)]} 0 1
[junit4] 2> 2316552 INFO (qtp913267724-48516) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-12 (1553882545472405505)]} 0 0
[junit4] 2> 2316552 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-12 (1553882545472405504)]} 0 3
[junit4] 2> 2316553 INFO (qtp1254808657-48513) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-12 (1553882545472405505)]} 0 3
[junit4] 2> 2316557 INFO (qtp913267724-48519) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-8 (-1553882545477648384)]} 0 1
[junit4] 2> 2316557 INFO (qtp913267724-48518) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-8 (-1553882545477648385)]} 0 1
[junit4] 2> 2316557 INFO (qtp1254808657-48523) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-8 (-1553882545477648384)]} 0 3
[junit4] 2> 2316558 INFO (qtp1254808657-48514) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-8 (-1553882545477648385)]} 0 3
[junit4] 2> 2316563 INFO (qtp913267724-48507) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-13 (1553882545482891264)]} 0 1
[junit4] 2> 2316563 INFO (qtp1254808657-48515) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-13 (1553882545482891264)]} 0 3
[junit4] 2> 2316563 INFO (qtp913267724-48520) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-13 (1553882545483939840)]} 0 1
[junit4] 2> 2316563 INFO (qtp1254808657-48521) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-13 (1553882545483939840)]} 0 3
[junit4] 2> 2316567 INFO (qtp913267724-48512) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-9 (-1553882545489182720)]} 0 1
[junit4] 2> 2316567 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-9 (-1553882545489182721)]} 0 1
[junit4] 2> 2316568 INFO (qtp1254808657-48509) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-9 (-1553882545489182720)]} 0 2
[junit4] 2> 2316568 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-9 (-1553882545489182721)]} 0 3
[junit4] 2> 2316572 INFO (qtp913267724-48516) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-14 (1553882545493377024)]} 0 1
[junit4] 2> 2316572 INFO (qtp913267724-48519) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-14 (1553882545494425600)]} 0 0
[junit4] 2> 2316572 INFO (qtp1254808657-48513) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-14 (1553882545493377024)]} 0 2
[junit4] 2> 2316572 INFO (qtp1254808657-48523) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-14 (1553882545494425600)]} 0 2
[junit4] 2> 2316577 INFO (qtp913267724-48518) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-10 (-1553882545499668480)]} 0 0
[junit4] 2> 2316577 INFO (qtp1254808657-48515) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-10 (-1553882545499668480)]} 0 2
[junit4] 2> 2316577 INFO (qtp913267724-48507) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-10 (-1553882545498619904)]} 0 1
[junit4] 2> 2316578 INFO (qtp1254808657-48514) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-10 (-1553882545498619904)]} 0 3
[junit4] 2> 2316581 INFO (qtp913267724-48520) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-15 (1553882545503862784)]} 0 1
[junit4] 2> 2316582 INFO (qtp1254808657-48521) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-15 (1553882545503862784)]} 0 2
[junit4] 2> 2316582 INFO (qtp913267724-48512) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-15 (1553882545503862785)]} 0 0
[junit4] 2> 2316582 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-15 (1553882545503862785)]} 0 2
[junit4] 2> 2316586 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-16 (1553882545508057088)]} 0 0
[junit4] 2> 2316586 INFO (qtp1254808657-48509) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-16 (1553882545508057088)]} 0 2
[junit4] 2> 2316587 INFO (qtp913267724-48516) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-16 (1553882545509105664)]} 0 1
[junit4] 2> 2316587 INFO (qtp1254808657-48523) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-16 (1553882545509105664)]} 0 3
[junit4] 2> 2316590 INFO (qtp913267724-48519) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-11 (-1553882545513299968)]} 0 0
[junit4] 2> 2316591 INFO (qtp1254808657-48513) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-11 (-1553882545513299968)]} 0 2
[junit4] 2> 2316592 INFO (qtp913267724-48518) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-11 (-1553882545514348544)]} 0 1
[junit4] 2> 2316592 INFO (qtp1254808657-48515) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-11 (-1553882545514348544)]} 0 3
[junit4] 2> 2316596 INFO (qtp913267724-48507) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-17 (1553882545518542848)]} 0 1
[junit4] 2> 2316596 INFO (qtp1254808657-48514) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-17 (1553882545518542848)]} 0 2
[junit4] 2> 2316597 INFO (qtp913267724-48520) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-17 (1553882545519591424)]} 0 0
[junit4] 2> 2316597 INFO (qtp1254808657-48521) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-17 (1553882545519591424)]} 0 2
[junit4] 2> 2316600 INFO (qtp913267724-48512) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-12 (-1553882545522737152)]} 0 0
[junit4] 2> 2316600 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-12 (-1553882545522737152)]} 0 2
[junit4] 2> 2316602 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-12 (-1553882545524834304)]} 0 1
[junit4] 2> 2316602 INFO (qtp1254808657-48509) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-12 (-1553882545524834304)]} 0 2
[junit4] 2> 2316604 INFO (qtp913267724-48516) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-18 (1553882545527980032)]} 0 0
[junit4] 2> 2316605 INFO (qtp1254808657-48523) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-18 (1553882545527980032)]} 0 2
[junit4] 2> 2316606 INFO (qtp913267724-48519) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-18 (1553882545529028608)]} 0 1
[junit4] 2> 2316607 INFO (qtp1254808657-48513) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-18 (1553882545529028608)]} 0 3
[junit4] 2> 2316609 INFO (qtp913267724-48518) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-19 (1553882545533222912)]} 0 1
[junit4] 2> 2316609 INFO (qtp1254808657-48515) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-19 (1553882545533222912)]} 0 2
[junit4] 2> 2316611 INFO (qtp913267724-48507) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-19 (1553882545534271488)]} 0 1
[junit4] 2> 2316611 INFO (qtp1254808657-48514) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-19 (1553882545534271488)]} 0 2
[junit4] 2> 2316613 INFO (qtp913267724-48520) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-20 (1553882545537417216)]} 0 0
[junit4] 2> 2316614 INFO (qtp1254808657-48521) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-20 (1553882545537417216)]} 0 2
[junit4] 2> 2316615 INFO (qtp913267724-48512) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-20 (1553882545539514368)]} 0 0
[junit4] 2> 2316616 INFO (qtp1254808657-48511) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-20 (1553882545539514368)]} 0 2
[junit4] 2> 2316618 INFO (qtp913267724-48517) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-21 (1553882545541611520)]} 0 1
[junit4] 2> 2316618 INFO (qtp1254808657-48509) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-21 (1553882545541611520)]} 0 3
[junit4] 2> 2316620 INFO (qtp913267724-48516) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-21 (1553882545543708672)]} 0 0
[junit4] 2> 2316621 INFO (qtp1254808657-48523) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-21 (1553882545543708672)]} 0 3
[junit4] 2> 2316623 INFO (qtp913267724-48519) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-15 (-1553882545546854400)]} 0 1
[junit4] 2> 2316623 INFO (qtp1254808657-48513) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-15 (-1553882545546854400)]} 0 2
[junit4] 2> 2316625 INFO (qtp913267724-48518) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-15 (-1553882545550000128)]} 0 0
[junit4] 2> 2316625 INFO (qtp1254808657-48515) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-15 (-1553882545550000128)]} 0 2
[junit4] 2> 2316627 INFO (qtp913267724-48507) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-22 (1553882545552097280)]} 0 0
[junit4] 2> 2316628 INFO (qtp1254808657-48514) [n:127.0.0.1:56154_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-22 (1553882545552097280)]} 0 2
[junit4] 2> 2316630 INFO (qtp913267724-48520) [n:127.0.0.1:40691_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56154/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-22 (1553882545554194432)]} 0 1
[junit4] 2> 2316630 INFO (qtp1254808657-48521) [n:127.0.0.1:56154_s
[...truncated too long message...]
unOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=2,free=59067384,total=518979584
[junit4] 2> NOTE: All tests run in this JVM: [BasicFunctionalityTest, NoCacheHeaderTest, TestFieldTypeCollectionResource, RankQueryTest, DistributedFacetExistsSmallTest, DefaultValueUpdateProcessorTest, TestJavabinTupleStreamParser, CollectionsAPIDistributedZkTest, DistributedFacetPivotSmallTest, SolrInfoMBeanTest, ForceLeaderTest, TestSchemaNameResource, BlockJoinFacetDistribTest, ConfigSetsAPITest, TestManagedStopFilterFactory, NotRequiredUniqueKeyTest, DistributedSuggestComponentTest, TestRandomFlRTGCloud, TestRTimerTree, ReplicaListTransformerTest, TestDocSet, TestSubQueryTransformerDistrib, TestSubQueryTransformer, HighlighterConfigTest, DirectUpdateHandlerOptimizeTest, StatsComponentTest, BlockCacheTest, RulesTest, ReturnFieldsTest, TestCloudSchemaless, CollectionReloadTest, TestJsonFacetRefinement, TestConfigSetProperties, TestDeleteCollectionOnDownNodes, TermVectorComponentTest, TestConfigSetsAPIZkFailure, TestTrie, TestCorePropertiesReload, SpellingQueryConverterTest, TestSerializedLuceneMatchVersion, TestSuggestSpellingConverter, TestCollapseQParserPlugin, TestRecoveryHdfs, CircularListTest, TestFastWriter, TestQueryUtils, TestSolrFieldCacheMBean, TestSolrDynamicMBean, TemplateUpdateProcessorTest, LukeRequestHandlerTest, SpatialHeatmapFacetsTest, WordBreakSolrSpellCheckerTest, TestSearcherReuse, TestExpandComponent, TestFreeTextSuggestions, TestMinMaxOnMultiValuedField, RegexBoostProcessorTest, BasicAuthIntegrationTest, CollectionTooManyReplicasTest, ConnectionReuseTest, LeaderInitiatedRecoveryOnShardRestartTest, CdcrBootstrapTest, SmileWriterTest, TestFieldCacheSanityChecker, TestTrackingShardHandlerFactory, TestHdfsCloudBackupRestore, TestRawResponseWriter, TestLMJelinekMercerSimilarityFactory, ShowFileRequestHandlerTest, TriLevelCompositeIdRoutingTest, TestJoin, ZkStateReaderTest, SolrCloudExampleTest, FileBasedSpellCheckerTest, CacheHeaderTest, TestSearchPerf, TestBadConfig, TestRealTimeGet, TestValueSourceCache, TestExactStatsCache, AliasIntegrationTest, TestPHPSerializedResponseWriter, TestDistributedMissingSort, HdfsThreadLeakTest, TestDynamicFieldCollectionResource, UUIDFieldTest, TestMiniSolrCloudClusterSSL, TestFieldCollectionResource, AnalyticsMergeStrategyTest, SolrIndexSplitterTest, TestManagedSchemaThreadSafety, TestDFRSimilarityFactory, TestShortCircuitedRequests, RAMDirectoryFactoryTest, PreAnalyzedFieldTest, TestJmxMonitoredMap, CdcrReplicationHandlerTest, TestHashQParserPlugin, DocValuesMissingTest, TestStressLiveNodes, TestRebalanceLeaders, DistributedTermsComponentTest, RequestLoggingTest, DistanceFunctionTest, TestSolrConfigHandlerConcurrent, TestScoreJoinQPNoScore, HLLUtilTest, BufferStoreTest, MBeansHandlerTest, TestMergePolicyConfig, EchoParamsTest, UpdateRequestProcessorFactoryTest, DebugComponentTest, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, InfoHandlerTest, TestSolrQueryResponse, SuggesterWFSTTest, TestCloudPseudoReturnFields, TestAtomicUpdateErrorCases, SolrIndexConfigTest, DistributedDebugComponentTest, ConvertedLegacyTest, TestJsonRequest, ExitableDirectoryReaderTest, AsyncCallRequestStatusResponseTest, SolrCmdDistributorTest, HdfsChaosMonkeyNothingIsSafeTest, FieldMutatingUpdateProcessorTest, HttpSolrCallGetCoreTest, TestCloudInspectUtil, AddSchemaFieldsUpdateProcessorFactoryTest, TestSimpleTrackingShardHandler, DocExpirationUpdateProcessorFactoryTest, TestEmbeddedSolrServerConstructors, TestFieldCacheWithThreads, TestSlowCompositeReaderWrapper, TestZkChroot, SmileWriterTest, ClassificationUpdateProcessorTest, TolerantUpdateProcessorTest, SpatialRPTFieldTypeTest, TestFilteredDocIdSet, TestOmitPositions, CleanupOldIndexTest, ResourceLoaderTest, AnalysisErrorHandlingTest, ParsingFieldUpdateProcessorsTest, TestAddFieldRealTimeGet, TestCopyFieldCollectionResource, UpdateParamsTest, TestPerFieldSimilarityClassic, TestDistributedGrouping, TestLocalFSCloudBackupRestore, LoggingHandlerTest, ClusterStateTest, TestUninvertingReader, SegmentsInfoRequestHandlerTest, TestIndexSearcher, HdfsRecoverLeaseTest, TestGeoJSONResponseWriter, TestConfigSetsAPI, TestAuthenticationFramework, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, LeaderElectionIntegrationTest, BasicZkTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestRandomFaceting, LeaderElectionTest, ZkSolrClientTest, AutoCommitTest, TestFaceting, TestRecovery, TermVectorComponentDistributedTest, TestReload, HardAutoCommitTest, TestRangeQuery, TestSolr4Spatial, SpellCheckComponentTest, TestGroupingSearch, QueryElevationComponentTest, PeerSyncTest, BadIndexSchemaTest, TestSort, TestFiltering, DirectUpdateHandlerTest, SoftAutoCommitTest, CurrencyFieldOpenExchangeTest, CurrencyFieldXmlFileTest, SimplePostToolTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, SignatureUpdateProcessorFactoryTest, TestExtendedDismaxParser, SpellCheckCollatorTest, CoreAdminHandlerTest, SolrRequestParserTest, TestFoldingMultitermQuery, DocValuesTest, SuggesterTSTTest, SuggesterTest, SolrCoreCheckLockOnStartupTest, TestUpdate, QueryEqualityTest, SortByFunctionTest, XsltUpdateRequestHandlerTest, TestSurroundQueryParser, StandardRequestHandlerTest, TestWriterPerf, TestReversedWildcardFilterFactory, TestQueryTypes, TermsComponentTest, RequestHandlersTest, DocumentBuilderTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, MoreLikeThisHandlerTest, FieldAnalysisRequestHandlerTest, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, IndexSchemaTest, TestCollationField, JSONWriterTest, TestCSVResponseWriter, QueryParsingTest, JsonLoaderTest, BinaryUpdateRequestHandlerTest, CSVRequestHandlerTest, SearchHandlerTest, TestBinaryResponseWriter, SOLR749Test, TestQuerySenderListener, AlternateDirectoryTest, TestSolrIndexConfig, TestQuerySenderNoQuery, CopyFieldTest, BadComponentTest, TestSolrDeletionPolicy2, SampleTest, TestBinaryField, NumericFieldsTest, MinimalSchemaTest, TestConfig, DirectSolrConnectionTest, TestLuceneMatchVersion, TestPhraseSuggestions, TestCharFilters, SynonymTokenizerTest, TestLMDirichletSimilarityFactory, TestFastOutputStream, URLClassifyProcessorTest, TestSystemIdResolver, TestUtils, SliceStateTest, FileUtilsTest, DistributedIntervalFacetingTest, CollectionStateFormat2Test, CollectionsAPIAsyncDistributedZkTest, ConcurrentDeleteAndCreateCollectionTest, ConnectionManagerTest, CreateCollectionCleanupTest, DeleteInactiveReplicaTest, DeleteNodeTest, DeleteShardTest, DistribJoinFromCollectionTest, DistributedVersionInfoTest, LeaderElectionContextKeyTest, LeaderFailureAfterFreshStartTest, MultiThreadedOCPTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, PeerSyncReplicationTest, SSLMigrationTest, SaslZkACLProviderTest, SharedFSAutoReplicaFailoverTest, SimpleCollectionCreateDeleteTest, TestCloudRecovery, TestClusterProperties, TestDownShardTolerantSearch, TestExclusionRuleCollectionAccess, TestLeaderElectionWithEmptyReplica, TestLockTree, TestMiniSolrCloudCluster, TestRequestForwarding, TestSSLRandomization, TestSegmentSorting, TestSizeLimitedDistributedMap, TestTolerantUpdateProcessorCloud, TlogReplayBufferedWhileIndexingTest, HdfsBasicDistributedZk2Test, HdfsBasicDistributedZkTest, HdfsCollectionsAPIDistributedZkTest, HdfsRecoveryZkTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=HdfsRecoveryZkTest -Dtests.seed=53EFA0D1A06992C1 -Dtests.slow=true -Dtests.locale=be-BY -Dtests.timezone=Atlantic/Azores -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J1 | HdfsRecoveryZkTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
[junit4] > at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
[junit4] > at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
[junit4] > at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
[junit4] > at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)
[junit4] > at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)
[junit4] > at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([53EFA0D1A06992C1]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] Completed [529/662 (1!)] on J1 in 54.99s, 1 test, 1 failure <<< FAILURES!
[...truncated 62856 lines...]
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 557 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/557/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130) at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102) at sun.reflect.GeneratedConstructorAccessor156.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
at sun.reflect.GeneratedConstructorAccessor156.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)
at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)
at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([7AC57E5DEEE16784]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
at sun.reflect.GeneratedMethodAccessor22.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12330 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7AC57E5DEEE16784-001/init-core-data-001
[junit4] 2> 2570965 INFO (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 2570965 INFO (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7AC57E5DEEE16784-001/tempDir-001
[junit4] 2> 2570966 INFO (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2570966 INFO (Thread-4901) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2570966 INFO (Thread-4901) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2571067 INFO (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:53613
[junit4] 2> 2571075 INFO (jetty-launcher-3729-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2571076 INFO (jetty-launcher-3729-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2571078 INFO (jetty-launcher-3729-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1237504b{/solr,null,AVAILABLE}
[junit4] 2> 2571079 INFO (jetty-launcher-3729-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@270fba61{/solr,null,AVAILABLE}
[junit4] 2> 2571086 INFO (jetty-launcher-3729-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@156f9155{HTTP/1.1,[http/1.1]}{127.0.0.1:34251}
[junit4] 2> 2571086 INFO (jetty-launcher-3729-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4f85895{HTTP/1.1,[http/1.1]}{127.0.0.1:61130}
[junit4] 2> 2571086 INFO (jetty-launcher-3729-thread-2) [ ] o.e.j.s.Server Started @2575610ms
[junit4] 2> 2571086 INFO (jetty-launcher-3729-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=61130}
[junit4] 2> 2571087 ERROR (jetty-launcher-3729-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2571087 INFO (jetty-launcher-3729-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 2571087 INFO (jetty-launcher-3729-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2571087 INFO (jetty-launcher-3729-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2571087 INFO (jetty-launcher-3729-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-16T04:11:21.804Z
[junit4] 2> 2571088 INFO (jetty-launcher-3729-thread-1) [ ] o.e.j.s.Server Started @2575612ms
[junit4] 2> 2571088 INFO (jetty-launcher-3729-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34251}
[junit4] 2> 2571088 ERROR (jetty-launcher-3729-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2571088 INFO (jetty-launcher-3729-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 2571088 INFO (jetty-launcher-3729-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2571088 INFO (jetty-launcher-3729-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2571089 INFO (jetty-launcher-3729-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-12-16T04:11:21.806Z
[junit4] 2> 2571092 INFO (jetty-launcher-3729-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2571093 INFO (jetty-launcher-3729-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2571102 INFO (jetty-launcher-3729-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2571103 INFO (jetty-launcher-3729-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53613/solr
[junit4] 2> 2571104 INFO (jetty-launcher-3729-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2571105 INFO (jetty-launcher-3729-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53613/solr
[junit4] 2> 2571137 INFO (jetty-launcher-3729-thread-2) [n:127.0.0.1:61130_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:61130_solr
[junit4] 2> 2571138 INFO (jetty-launcher-3729-thread-1) [n:127.0.0.1:34251_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34251_solr
[junit4] 2> 2571139 INFO (jetty-launcher-3729-thread-2) [n:127.0.0.1:61130_solr ] o.a.s.c.Overseer Overseer (id=97115274063708166-127.0.0.1:61130_solr-n_0000000000) starting
[junit4] 2> 2571154 INFO (zkCallback-3740-thread-1-processing-n:127.0.0.1:61130_solr) [n:127.0.0.1:61130_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2571155 INFO (zkCallback-3739-thread-1-processing-n:127.0.0.1:34251_solr) [n:127.0.0.1:34251_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2571170 INFO (jetty-launcher-3729-thread-2) [n:127.0.0.1:61130_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61130_solr
[junit4] 2> 2571177 INFO (zkCallback-3739-thread-1-processing-n:127.0.0.1:34251_solr) [n:127.0.0.1:34251_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2571178 INFO (zkCallback-3740-thread-1-processing-n:127.0.0.1:61130_solr) [n:127.0.0.1:61130_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2571253 INFO (jetty-launcher-3729-thread-2) [n:127.0.0.1:61130_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7AC57E5DEEE16784-001/tempDir-001/node2/.
[junit4] 2> 2571303 INFO (jetty-launcher-3729-thread-1) [n:127.0.0.1:34251_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7AC57E5DEEE16784-001/tempDir-001/node1/.
[junit4] 2> 2571312 INFO (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2571314 INFO (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53613/solr ready
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 2571359 WARN (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 2571378 WARN (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2571382 INFO (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2571401 INFO (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_solaris.vm_44617_hdfs____ew41da/webapp
[junit4] 2> 2571934 INFO (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:44617
[junit4] 2> 2572044 WARN (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2572045 INFO (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2572059 INFO (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_51266_datanode____.2c6fgj/webapp
[junit4] 2> 2572560 INFO (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:51266
[junit4] 2> 2572647 WARN (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2572648 INFO (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2572674 INFO (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_35699_datanode____.ksa1np/webapp
[junit4] 2> 2572866 INFO (IPC Server handler 4 on 48543) [ ] BlockStateChange BLOCK* processReport: from storage DS-b48b5cf9-de75-4d45-93fd-aa2d159fa9df node DatanodeRegistration(127.0.0.1:53346, datanodeUuid=f5f2ae84-2144-4f69-a188-79f47fac49ab, infoPort=55659, infoSecurePort=0, ipcPort=40250, storageInfo=lv=-56;cid=testClusterID;nsid=188686613;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2572866 INFO (IPC Server handler 4 on 48543) [ ] BlockStateChange BLOCK* processReport: from storage DS-ec5f76b6-5d94-4ddb-b612-37ac1fe74b2f node DatanodeRegistration(127.0.0.1:53346, datanodeUuid=f5f2ae84-2144-4f69-a188-79f47fac49ab, infoPort=55659, infoSecurePort=0, ipcPort=40250, storageInfo=lv=-56;cid=testClusterID;nsid=188686613;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2573238 INFO (SUITE-HdfsRecoveryZkTest-seed#[7AC57E5DEEE16784]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35699
[junit4] 2> 2573400 INFO (IPC Server handler 6 on 48543) [ ] BlockStateChange BLOCK* processReport: from storage DS-bbd66215-9c1e-478e-b312-bfd4a2509924 node DatanodeRegistration(127.0.0.1:36891, datanodeUuid=796152a9-1319-426d-8aaf-562f15601982, infoPort=51965, infoSecurePort=0, ipcPort=56912, storageInfo=lv=-56;cid=testClusterID;nsid=188686613;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2573400 INFO (IPC Server handler 6 on 48543) [ ] BlockStateChange BLOCK* processReport: from storage DS-96220073-480e-4ef8-b8bf-a7b1fb8abf10 node DatanodeRegistration(127.0.0.1:36891, datanodeUuid=796152a9-1319-426d-8aaf-562f15601982, infoPort=51965, infoSecurePort=0, ipcPort=56912, storageInfo=lv=-56;cid=testClusterID;nsid=188686613;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2573587 INFO (TEST-HdfsRecoveryZkTest.test-seed#[7AC57E5DEEE16784]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2573591 INFO (qtp118349728-20566) [n:127.0.0.1:61130_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2573596 INFO (OverseerThreadFactory-9593-thread-1-processing-n:127.0.0.1:61130_solr) [n:127.0.0.1:61130_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 2573712 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 2573712 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 2573823 INFO (zkCallback-3740-thread-2-processing-n:127.0.0.1:61130_solr) [n:127.0.0.1:61130_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2573823 INFO (zkCallback-3739-thread-1-processing-n:127.0.0.1:34251_solr) [n:127.0.0.1:34251_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2574732 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 2574733 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 2574759 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema [recoverytest_shard1_replica1] Schema name=minimal
[junit4] 2> 2574759 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema [recoverytest_shard1_replica2] Schema name=minimal
[junit4] 2> 2574762 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2574762 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2574765 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica1' using configuration from collection recoverytest
[junit4] 2> 2574765 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica2' using configuration from collection recoverytest
[junit4] 2> 2574772 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:48543/data
[junit4] 2> 2574772 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 2574772 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [[recoverytest_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7AC57E5DEEE16784-001/tempDir-001/node1/recoverytest_shard1_replica1], dataDir=[hdfs://solaris-vm:48543/data/recoverytest/core_node1/data/]
[junit4] 2> 2574772 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:48543/data/recoverytest/core_node1/data/snapshot_metadata
[junit4] 2> 2574772 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:48543/data
[junit4] 2> 2574772 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 2574772 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [[recoverytest_shard1_replica2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_7AC57E5DEEE16784-001/tempDir-001/node2/recoverytest_shard1_replica2], dataDir=[hdfs://solaris-vm:48543/data/recoverytest/core_node2/data/]
[junit4] 2> 2574772 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:48543/data/recoverytest/core_node2/data/snapshot_metadata
[junit4] 2> 2574783 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2574783 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2574783 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2574783 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2574783 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2574783 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2574789 WARN (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 2574789 WARN (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 2579826 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2579831 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:48543/data/recoverytest/core_node1/data
[junit4] 2> 2579831 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2579842 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:48543/data/recoverytest/core_node2/data
[junit4] 2> 2579858 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:48543/data/recoverytest/core_node1/data/index
[junit4] 2> 2579865 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:48543/data/recoverytest/core_node2/data/index
[junit4] 2> 2579868 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2579868 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2579868 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2579873 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2579873 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2579873 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2579878 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2579884 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2579944 INFO (IPC Server handler 5 on 48543) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:53346 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-96220073-480e-4ef8-b8bf-a7b1fb8abf10:NORMAL:127.0.0.1:36891|RBW], ReplicaUC[[DISK]DS-b48b5cf9-de75-4d45-93fd-aa2d159fa9df:NORMAL:127.0.0.1:53346|FINALIZED]]} size 0
[junit4] 2> 2579944 INFO (IPC Server handler 5 on 48543) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:53346 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-96220073-480e-4ef8-b8bf-a7b1fb8abf10:NORMAL:127.0.0.1:36891|RBW], ReplicaUC[[DISK]DS-ec5f76b6-5d94-4ddb-b612-37ac1fe74b2f:NORMAL:127.0.0.1:53346|FINALIZED]]} size 0
[junit4] 2> 2579944 INFO (IPC Server handler 5 on 48543) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:36891 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-b48b5cf9-de75-4d45-93fd-aa2d159fa9df:NORMAL:127.0.0.1:53346|FINALIZED], ReplicaUC[[DISK]DS-bbd66215-9c1e-478e-b312-bfd4a2509924:NORMAL:127.0.0.1:36891|FINALIZED]]} size 0
[junit4] 2> 2579946 INFO (IPC Server handler 7 on 48543) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:36891 is added to blk_1073741825_1001 size 71
[junit4] 2> 2579963 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 2579963 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2579963 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 2579975 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2579975 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2579976 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 2579976 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2579976 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 2579992 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@f6130c5[recoverytest_shard1_replica2] main]
[junit4] 2> 2579993 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2579993 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2579994 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2579995 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2579995 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2579995 INFO (searcherExecutor-9599-thread-1-processing-n:127.0.0.1:61130_solr x:recoverytest_shard1_replica2 s:shard1 c:recoverytest) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [recoverytest_shard1_replica2] Registered new searcher Searcher@f6130c5[recoverytest_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2579995 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1553844394484826112
[junit4] 2> 2580005 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 2580007 INFO (zkCallback-3740-thread-1-processing-n:127.0.0.1:61130_solr) [n:127.0.0.1:61130_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2580007 INFO (zkCallback-3739-thread-1-processing-n:127.0.0.1:34251_solr) [n:127.0.0.1:34251_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2580013 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7753ee6c[recoverytest_shard1_replica1] main]
[junit4] 2> 2580014 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2580015 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2580015 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2580015 INFO (searcherExecutor-9598-thread-1-processing-n:127.0.0.1:34251_solr x:recoverytest_shard1_replica1 s:shard1 c:recoverytest) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [recoverytest_shard1_replica1] Registered new searcher Searcher@7753ee6c[recoverytest_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2580015 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1553844394505797632
[junit4] 2> 2580510 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2580511 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2580511 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/
[junit4] 2> 2580511 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:61130/solr START replicas=[http://127.0.0.1:34251/solr/recoverytest_shard1_replica1/] nUpdates=100
[junit4] 2> 2580511 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:61130/solr DONE. We have no versions. sync failed.
[junit4] 2> 2580514 INFO (qtp8806061-20560) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.S.Request [recoverytest_shard1_replica1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 2580515 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 2580515 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 2580521 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/ shard1
[junit4] 2> 2580626 INFO (zkCallback-3740-thread-1-processing-n:127.0.0.1:61130_solr) [n:127.0.0.1:61130_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2580627 INFO (zkCallback-3739-thread-1-processing-n:127.0.0.1:34251_solr) [n:127.0.0.1:34251_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2580672 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2580675 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=6969
[junit4] 2> 2580779 INFO (zkCallback-3740-thread-2-processing-n:127.0.0.1:61130_solr) [n:127.0.0.1:61130_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2580779 INFO (zkCallback-3739-thread-1-processing-n:127.0.0.1:34251_solr) [n:127.0.0.1:34251_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2581025 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=7316
[junit4] 2> 2581030 INFO (qtp118349728-20566) [n:127.0.0.1:61130_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2581131 INFO (zkCallback-3740-thread-3-processing-n:127.0.0.1:61130_solr) [n:127.0.0.1:61130_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2581131 INFO (zkCallback-3739-thread-1-processing-n:127.0.0.1:34251_solr) [n:127.0.0.1:34251_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2581604 INFO (OverseerCollectionConfigSetProcessor-97115274063708166-127.0.0.1:61130_solr-n_0000000000) [n:127.0.0.1:61130_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2582033 INFO (qtp118349728-20566) [n:127.0.0.1:61130_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=8446
[junit4] 2> 2582035 INFO (TEST-HdfsRecoveryZkTest.test-seed#[7AC57E5DEEE16784]) [ ] o.a.s.c.RecoveryZkTest Indexing 1350 documents
[junit4] 2> 2582079 INFO (qtp8806061-20558) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-0 (1553844396627066880)]} 0 23
[junit4] 2> 2582080 INFO (qtp118349728-20569) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 (1553844396627066880)]} 0 41
[junit4] 2> 2582084 INFO (qtp8806061-20554) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-0 (1553844396628115456)]} 0 28
[junit4] 2> 2582085 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 (1553844396628115456)]} 0 45
[junit4] 2> 2582087 INFO (qtp8806061-20560) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-0 (-1553844396673204224)]} 0 2
[junit4] 2> 2582087 INFO (qtp118349728-20568) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-0 (-1553844396673204224)]} 0 5
[junit4] 2> 2582093 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-0 (-1553844396679495680)]} 0 2
[junit4] 2> 2582093 INFO (qtp118349728-20575) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-0 (-1553844396679495680)]} 0 4
[junit4] 2> 2582094 INFO (qtp8806061-20559) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-1 (1553844396680544256)]} 0 2
[junit4] 2> 2582094 INFO (qtp118349728-20576) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 (1553844396680544256)]} 0 4
[junit4] 2> 2582099 INFO (qtp8806061-20561) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-1 (1553844396686835712)]} 0 2
[junit4] 2> 2582100 INFO (qtp118349728-20566) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 (1553844396686835712)]} 0 5
[junit4] 2> 2582101 INFO (qtp8806061-20557) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-2 (1553844396687884288)]} 0 3
[junit4] 2> 2582102 INFO (qtp118349728-20564) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 (1553844396687884288)]} 0 5
[junit4] 2> 2582109 INFO (qtp8806061-20554) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-3 (1553844396696272896)]} 0 3
[junit4] 2> 2582109 INFO (qtp8806061-20558) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-2 (1553844396694175744)]} 0 5
[junit4] 2> 2582109 INFO (qtp118349728-20569) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 (1553844396694175744)]} 0 6
[junit4] 2> 2582109 INFO (qtp118349728-20564) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 (1553844396696272896)]} 0 5
[junit4] 2> 2582115 INFO (qtp8806061-20560) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-3 (1553844396703612928)]} 0 2
[junit4] 2> 2582115 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-4 (1553844396703612929)]} 0 2
[junit4] 2> 2582116 INFO (qtp118349728-20575) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 (1553844396703612929)]} 0 4
[junit4] 2> 2582116 INFO (qtp118349728-20568) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 (1553844396703612928)]} 0 5
[junit4] 2> 2582123 INFO (qtp8806061-20561) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-4 (1553844396710952961)]} 0 3
[junit4] 2> 2582123 INFO (qtp8806061-20559) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-1 (-1553844396710952960)]} 0 3
[junit4] 2> 2582123 INFO (qtp118349728-20566) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 (1553844396710952961)]} 0 5
[junit4] 2> 2582123 INFO (qtp118349728-20576) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-1 (-1553844396710952960)]} 0 5
[junit4] 2> 2582130 INFO (qtp8806061-20557) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-5 (1553844396718292993)]} 0 3
[junit4] 2> 2582130 INFO (qtp8806061-20554) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-1 (-1553844396718292992)]} 0 3
[junit4] 2> 2582130 INFO (qtp118349728-20569) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 (1553844396718292993)]} 0 5
[junit4] 2> 2582130 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-1 (-1553844396718292992)]} 0 5
[junit4] 2> 2582136 INFO (qtp8806061-20558) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-2 (-1553844396725633025)]} 0 2
[junit4] 2> 2582137 INFO (qtp118349728-20575) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-2 (-1553844396725633025)]} 0 4
[junit4] 2> 2582137 INFO (qtp8806061-20560) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-5 (1553844396725633024)]} 0 3
[junit4] 2> 2582138 INFO (qtp118349728-20564) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 (1553844396725633024)]} 0 5
[junit4] 2> 2582145 INFO (qtp8806061-20561) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-2 (-1553844396732973057)]} 0 3
[junit4] 2> 2582145 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-6 (1553844396732973056)]} 0 5
[junit4] 2> 2582145 INFO (qtp118349728-20568) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 (1553844396732973056)]} 0 6
[junit4] 2> 2582146 INFO (qtp118349728-20566) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-2 (-1553844396732973057)]} 0 6
[junit4] 2> 2582152 INFO (qtp8806061-20559) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-6 (1553844396742410241)]} 0 1
[junit4] 2> 2582152 INFO (qtp118349728-20569) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-6 (1553844396742410241)]} 0 3
[junit4] 2> 2582158 INFO (qtp8806061-20557) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-7 (1553844396742410240)]} 0 2
[junit4] 2> 2582160 INFO (qtp118349728-20576) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 (1553844396742410240)]} 0 12
[junit4] 2> 2582161 INFO (qtp8806061-20554) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-7 (1553844396750798848)]} 0 3
[junit4] 2> 2582161 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-7 (1553844396750798848)]} 0 5
[junit4] 2> 2582169 INFO (qtp8806061-20560) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-8 (1553844396757090304)]} 0 2
[junit4] 2> 2582169 INFO (qtp8806061-20558) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-8 (1553844396758138880)]} 0 2
[junit4] 2> 2582169 INFO (qtp118349728-20575) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 (1553844396757090304)]} 0 6
[junit4] 2> 2582169 INFO (qtp118349728-20564) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-8 (1553844396758138880)]} 0 6
[junit4] 2> 2582182 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-9 (1553844396766527489)]} 0 6
[junit4] 2> 2582182 INFO (qtp8806061-20561) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-9 (1553844396766527488)]} 0 7
[junit4] 2> 2582182 INFO (qtp118349728-20568) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 (1553844396766527488)]} 0 11
[junit4] 2> 2582182 INFO (qtp118349728-20566) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-9 (1553844396766527489)]} 0 11
[junit4] 2> 2582194 INFO (qtp8806061-20559) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-5 (-1553844396783304704)]} 0 4
[junit4] 2> 2582194 INFO (qtp8806061-20557) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-5 (-1553844396783304705)]} 0 4
[junit4] 2> 2582194 INFO (qtp118349728-20569) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-5 (-1553844396783304704)]} 0 6
[junit4] 2> 2582194 INFO (qtp118349728-20576) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-5 (-1553844396783304705)]} 0 6
[junit4] 2> 2582205 INFO (qtp8806061-20560) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-10 (1553844396792741889)]} 0 1
[junit4] 2> 2582205 INFO (qtp8806061-20554) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-10 (1553844396792741888)]} 0 2
[junit4] 2> 2582206 INFO (qtp118349728-20575) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 (1553844396792741888)]} 0 9
[junit4] 2> 2582206 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-10 (1553844396792741889)]} 0 9
[junit4] 2> 2582212 INFO (qtp8806061-20558) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-11 (1553844396805324801)]} 0 1
[junit4] 2> 2582212 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-11 (1553844396805324800)]} 0 1
[junit4] 2> 2582212 INFO (qtp118349728-20568) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-11 (1553844396805324801)]} 0 4
[junit4] 2> 2582212 INFO (qtp118349728-20564) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-11 (1553844396805324800)]} 0 5
[junit4] 2> 2582218 INFO (qtp8806061-20559) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-6 (-1553844396811616257)]} 0 0
[junit4] 2> 2582218 INFO (qtp8806061-20561) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-6 (-1553844396811616256)]} 0 1
[junit4] 2> 2582218 INFO (qtp118349728-20569) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-6 (-1553844396811616257)]} 0 3
[junit4] 2> 2582218 INFO (qtp118349728-20566) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-6 (-1553844396811616256)]} 0 3
[junit4] 2> 2582229 INFO (qtp8806061-20560) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-12 (1553844396821053440)]} 0 1
[junit4] 2> 2582229 INFO (qtp8806061-20557) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-12 (1553844396821053441)]} 0 2
[junit4] 2> 2582229 INFO (qtp118349728-20575) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-12 (1553844396821053441)]} 0 5
[junit4] 2> 2582229 INFO (qtp118349728-20576) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-12 (1553844396821053440)]} 0 6
[junit4] 2> 2582235 INFO (qtp8806061-20558) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-13 (1553844396829442048)]} 0 1
[junit4] 2> 2582235 INFO (qtp8806061-20554) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-13 (1553844396830490624)]} 0 1
[junit4] 2> 2582235 INFO (qtp118349728-20568) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-13 (1553844396829442048)]} 0 3
[junit4] 2> 2582235 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-13 (1553844396830490624)]} 0 3
[junit4] 2> 2582241 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-14 (1553844396835733505)]} 0 1
[junit4] 2> 2582241 INFO (qtp118349728-20569) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-14 (1553844396835733505)]} 0 4
[junit4] 2> 2582241 INFO (qtp8806061-20559) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-14 (1553844396835733504)]} 0 1
[junit4] 2> 2582242 INFO (qtp118349728-20564) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-14 (1553844396835733504)]} 0 4
[junit4] 2> 2582246 INFO (qtp8806061-20561) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-15 (1553844396842024960)]} 0 1
[junit4] 2> 2582246 INFO (qtp8806061-20560) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-15 (1553844396842024961)]} 0 1
[junit4] 2> 2582246 INFO (qtp118349728-20575) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-15 (1553844396842024961)]} 0 3
[junit4] 2> 2582247 INFO (qtp118349728-20566) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-15 (1553844396842024960)]} 0 3
[junit4] 2> 2582255 INFO (qtp8806061-20557) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-8 (-1553844396852510720)]} 0 1
[junit4] 2> 2582255 INFO (qtp8806061-20558) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-8 (-1553844396851462144)]} 0 1
[junit4] 2> 2582256 INFO (qtp118349728-20568) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-8 (-1553844396852510720)]} 0 2
[junit4] 2> 2582256 INFO (qtp118349728-20576) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-8 (-1553844396851462144)]} 0 3
[junit4] 2> 2582261 INFO (qtp8806061-20554) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-16 (1553844396857753600)]} 0 1
[junit4] 2> 2582262 INFO (qtp8806061-20559) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-16 (1553844396857753601)]} 0 2
[junit4] 2> 2582262 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-16 (1553844396857753600)]} 0 4
[junit4] 2> 2582262 INFO (qtp118349728-20569) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-16 (1553844396857753601)]} 0 4
[junit4] 2> 2582267 INFO (qtp8806061-20561) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-9 (-1553844396864045057)]} 0 1
[junit4] 2> 2582268 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-9 (-1553844396864045056)]} 0 1
[junit4] 2> 2582268 INFO (qtp118349728-20564) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-9 (-1553844396864045056)]} 0 3
[junit4] 2> 2582268 INFO (qtp118349728-20575) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-9 (-1553844396864045057)]} 0 3
[junit4] 2> 2582273 INFO (qtp8806061-20557) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-17 (1553844396870336512)]} 0 1
[junit4] 2> 2582273 INFO (qtp8806061-20560) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-17 (1553844396871385088)]} 0 1
[junit4] 2> 2582273 INFO (qtp118349728-20566) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-17 (1553844396870336512)]} 0 3
[junit4] 2> 2582274 INFO (qtp118349728-20568) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-17 (1553844396871385088)]} 0 3
[junit4] 2> 2582278 INFO (qtp8806061-20554) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-18 (1553844396876627968)]} 0 1
[junit4] 2> 2582279 INFO (qtp8806061-20558) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-18 (1553844396875579392)]} 0 1
[junit4] 2> 2582279 INFO (qtp118349728-20569) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-18 (1553844396876627968)]} 0 2
[junit4] 2> 2582279 INFO (qtp118349728-20576) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-18 (1553844396875579392)]} 0 3
[junit4] 2> 2582283 INFO (qtp8806061-20559) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-13 (-1553844396881870848)]} 0 0
[junit4] 2> 2582283 INFO (qtp8806061-20561) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-13 (-1553844396881870849)]} 0 1
[junit4] 2> 2582284 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-13 (-1553844396881870848)]} 0 3
[junit4] 2> 2582284 INFO (qtp118349728-20564) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-13 (-1553844396881870849)]} 0 3
[junit4] 2> 2582288 INFO (qtp8806061-20557) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-19 (1553844396887113728)]} 0 1
[junit4] 2> 2582289 INFO (qtp118349728-20566) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-19 (1553844396887113728)]} 0 3
[junit4] 2> 2582289 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-19 (1553844396886065152)]} 0 2
[junit4] 2> 2582289 INFO (qtp118349728-20575) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-19 (1553844396886065152)]} 0 4
[junit4] 2> 2582293 INFO (qtp8806061-20560) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-15 (-1553844396892356608)]} 0 1
[junit4] 2> 2582293 INFO (qtp118349728-20568) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-15 (-1553844396892356608)]} 0 2
[junit4] 2> 2582294 INFO (qtp8806061-20554) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-15 (-1553844396892356609)]} 0 1
[junit4] 2> 2582294 INFO (qtp118349728-20569) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-15 (-1553844396892356609)]} 0 2
[junit4] 2> 2582298 INFO (qtp8806061-20558) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-20 (1553844396896550912)]} 0 1
[junit4] 2> 2582298 INFO (qtp118349728-20576) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-20 (1553844396896550912)]} 0 3
[junit4] 2> 2582300 INFO (qtp8806061-20559) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-20 (1553844396898648064)]} 0 1
[junit4] 2> 2582301 INFO (qtp118349728-20565) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-20 (1553844396898648064)]} 0 3
[junit4] 2> 2582303 INFO (qtp8806061-20561) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-21 (1553844396902842368)]} 0 1
[junit4] 2> 2582303 INFO (qtp118349728-20564) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-21 (1553844396902842368)]} 0 2
[junit4] 2> 2582306 INFO (qtp8806061-20557) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-21 (1553844396903890944)]} 0 1
[junit4] 2> 2582306 INFO (qtp118349728-20566) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-21 (1553844396903890944)]} 0 4
[junit4] 2> 2582313 INFO (qtp8806061-20556) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-16 (-1553844396907036672)]} 0 5
[junit4] 2> 2582313 INFO (qtp118349728-20575) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-16 (-1553844396907036672)]} 0 7
[junit4] 2> 2582316 INFO (qtp8806061-20560) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-16 (-1553844396915425280)]} 0 1
[junit4] 2> 2582316 INFO (qtp118349728-20568) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-16 (-1553844396915425280)]} 0 2
[junit4] 2> 2582317 INFO (qtp8806061-20554) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-22 (1553844396917522432)]} 0 1
[junit4] 2> 2582317 INFO (qtp118349728-20575) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-22 (1553844396917522432)]} 0 2
[junit4] 2> 2582322 INFO (qtp8806061-20558) [n:127.0.0.1:34251_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:61130/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-22 (1553844396920668160)]} 0 1
[junit4] 2> 2582323 INFO (qtp118349728-20576) [n:127.0.0.1:61130_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-22 (1553844396920668160)]} 0 5
[junit4] 2> 2582325 INFO (qtp8806061-20559) [n:127.0.0.1:34251_solr c:recoverytest
[...truncated too long message...]
e
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {rnd_b=PostingsFormat(name=MockRandom), _version_=PostingsFormat(name=LuceneVarGapDocFreqInterval), a_t=PostingsFormat(name=LuceneVarGapDocFreqInterval), a_i=PostingsFormat(name=MockRandom), id=PostingsFormat(name=MockRandom)}, docValues:{}, maxPointsInLeafNode=81, maxMBSortInHeap=6.184696003617108, sim=RandomSimilarity(queryNorm=false,coord=yes): {}, locale=uk-UA, timezone=Europe/Tallinn
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=2,free=294715344,total=517996544
[junit4] 2> NOTE: All tests run in this JVM: [DirectSolrConnectionTest, PluginInfoTest, PKIAuthenticationIntegrationTest, MultiTermTest, AddBlockUpdateTest, VMParamsZkACLAndCredentialsProvidersTest, TestCloudPivotFacet, BooleanFieldTest, CdcrReplicationDistributedZkTest, TestLeaderElectionWithEmptyReplica, DeleteStatusTest, StatelessScriptUpdateProcessorFactoryTest, TestXmlQParserPlugin, CollectionsAPIAsyncDistributedZkTest, TestFileDictionaryLookup, BJQParserTest, TestLRUStatsCache, TestCharFilters, TestRestManager, HdfsSyncSliceTest, TestSha256AuthenticationProvider, TestDefaultSearchFieldResource, DOMUtilTest, TestLazyCores, TestSolrQueryParserDefaultOperatorResource, TestStressReorder, ChaosMonkeyNothingIsSafeTest, RecoveryAfterSoftCommitTest, DirectoryFactoryTest, PrimitiveFieldTypeTest, XsltUpdateRequestHandlerTest, TestJsonFacets, TestRandomRequestDistribution, TestPartialUpdateDeduplication, SystemInfoHandlerTest, TestPhraseSuggestions, DirectSolrSpellCheckerTest, DeleteReplicaTest, ShufflingReplicaListTransformerTest, JSONWriterTest, ShardSplitTest, CustomCollectionTest, CachingDirectoryFactoryTest, SOLR749Test, TestStressVersions, TestStressRecovery, SolrTestCaseJ4Test, SSLMigrationTest, TestExclusionRuleCollectionAccess, XmlUpdateRequestHandlerTest, TestWordDelimiterFilterFactory, TestRequestStatusCollectionAPI, SoftAutoCommitTest, TestPKIAuthenticationPlugin, HdfsDirectoryFactoryTest, TestStressLucene, HardAutoCommitTest, TestSQLHandler, TestTolerantUpdateProcessorCloud, StatsReloadRaceTest, TestTrieFacet, TestHdfsBackupRestoreCore, PeerSyncTest, HdfsChaosMonkeySafeLeaderTest, TestReplicaProperties, LeaderFailoverAfterPartitionTest, OverseerTest, SyncSliceTest, TestCollationField, CollectionStateFormat2Test, TestSolrIndexConfig, TestBackupRepositoryFactory, TestFastOutputStream, TestExportWriter, UnloadDistributedZkTest, PrimUtilsTest, SolrCoreTest, TestImplicitCoreProperties, SimpleFacetsTest, ResponseHeaderTest, TestLeaderInitiatedRecoveryThread, BlockDirectoryTest, TestXIncludeConfig, IndexSchemaRuntimeFieldTest, TestBM25SimilarityFactory, TestSchemaVersionResource, HighlighterTest, QueryEqualityTest, TestHighlightDedupGrouping, SuggestComponentContextFilterQueryTest, SampleTest, TestHashPartitioner, MigrateRouteKeyTest, CloudMLTQParserTest, TestElisionMultitermQuery, TestSizeLimitedDistributedMap, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestMacroExpander, AnalysisAfterCoreReloadTest, DistributedExpandComponentTest, TestSolr4Spatial2, GraphQueryTest, TestGraphTermsQParserPlugin, OutOfBoxZkACLAndCredentialsProvidersTest, UtilsToolTest, BlobRepositoryCloudTest, TestUniqueKeyFieldResource, TestDynamicFieldResource, TestSort, SparseHLLTest, TestSolrJ, TestReqParamsAPI, ZkStateWriterTest, TestFieldCacheVsDocValues, BlockJoinFacetRandomTest, TestChildDocTransformer, JSONWriterTest, MergeStrategyTest, EnumFieldTest, TestGraphMLResponseWriter, TestReRankQParserPlugin, OverseerTaskQueueTest, DistributedSpellCheckComponentTest, TestJettySolrRunner, FacetPivotSmallTest, TestFieldResource, NumericFieldsTest, HdfsNNFailoverTest, TlogReplayBufferedWhileIndexingTest, TestBinaryResponseWriter, DistributedQueueTest, TestDocumentBuilder, TestReloadAndDeleteDocs, TestGroupingSearch, SortSpecParsingTest, TestCoreContainer, ConnectionManagerTest, SharedFSAutoReplicaFailoverUtilsTest, TestAnalyzeInfixSuggestions, TestComponentsName, OverseerCollectionConfigSetProcessorTest, PeerSyncWithIndexFingerprintCachingTest, TestStandardQParsers, HdfsTlogReplayBufferedWhileIndexingTest, DisMaxRequestHandlerTest, SolrRequestParserTest, TestSolrXml, TestSolrCoreSnapshots, TestLFUCache, SolrPluginUtilsTest, PreAnalyzedFieldManagedSchemaCloudTest, DistributedFacetPivotSmallAdvancedTest, TestUpdate, TestManagedResource, TestPseudoReturnFields, TestSolrConfigHandlerCloud, TestCSVLoader, TestFaceting, TestExceedMaxTermLength, TestDocTermOrds, CloudExitableDirectoryReaderTest, RuleEngineTest, TestReplicationHandler, TestRandomFaceting, TestZkChroot, ZkCLITest, TestDistributedGrouping, ZkControllerTest, DistributedTermsComponentTest, TestSolr4Spatial, StatsComponentTest, SpellCheckComponentTest, QueryElevationComponentTest, ConvertedLegacyTest, TestFunctionQuery, BasicFunctionalityTest, DirectUpdateHandlerTest, DistributedQueryElevationComponentTest, CurrencyFieldXmlFileTest, SimplePostToolTest, TestCoreDiscovery, TestExtendedDismaxParser, SuggesterFSTTest, CoreAdminHandlerTest, TestFoldingMultitermQuery, DocValuesTest, SuggesterTest, TestTrie, SpatialFilterTest, SuggesterWFSTTest, PolyFieldTest, SchemaVersionSpecificBehaviorTest, DirectUpdateHandlerOptimizeTest, TestRemoteStreaming, DocValuesMultiTest, DistanceFunctionTest, TestSolrDeletionPolicy1, SolrInfoMBeanTest, CacheHeaderTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, LukeRequestHandlerTest, TestQueryUtils, StandardRequestHandlerTest, TestWriterPerf, DocumentAnalysisRequestHandlerTest, TestQueryTypes, TestOmitPositions, FileBasedSpellCheckerTest, TermsComponentTest, DocumentBuilderTest, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, TestIndexingPerformance, MoreLikeThisHandlerTest, RequiredFieldsTest, FastVectorHighlighterTest, LoggingHandlerTest, TestJmxIntegration, ReturnFieldsTest, TestCSVResponseWriter, MBeansHandlerTest, TestAnalyzedSuggestions, UniqFieldsUpdateProcessorFactoryTest, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, PingRequestHandlerTest, CSVRequestHandlerTest, TestBinaryResponseWriter, HighlighterConfigTest, UpdateParamsTest, AlternateDirectoryTest, ResponseLogComponentTest, SolrIndexConfigTest, BadComponentTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, TestDocSet, TestBinaryField, TestSearchPerf, MinimalSchemaTest, TestConfig, OutputWriterTest, TestFuzzyAnalyzedSuggestions, TestSolrCoreProperties, TestPostingsSolrHighlighter, SpellPossibilityIteratorTest, TestCodecSupport, TestLMDirichletSimilarityFactory, TestIBSimilarityFactory, TestJmxMonitoredMap, TimeZoneUtilsTest, TestFastLRUCache, TestSuggestSpellingConverter, DateFieldTest, ClusterStateTest, ZkNodePropsTest, FileUtilsTest, DistributedMLTComponentTest, DistributedIntervalFacetingTest, TestTolerantSearch, ActionThrottleTest, AssignTest, AsyncCallRequestStatusResponseTest, CleanupOldIndexTest, CollectionReloadTest, CollectionsAPISolrJTest, DistribDocExpirationUpdateProcessorTest, DocValuesNotIndexedTest, HttpPartitionTest, OverseerStatusTest, ReplaceNodeTest, SharedFSAutoReplicaFailoverTest, SolrCLIZkUtilsTest, SolrXmlInZkTest, TestCloudInspectUtil, TestCloudRecovery, TestLeaderElectionZkExpiry, TestLockTree, TestMiniSolrCloudCluster, TestRandomFlRTGCloud, TestSegmentSorting, TestSolrCloudWithDelegationTokens, TestStressCloudBlindAtomicUpdates, HdfsBasicDistributedZkTest, HdfsCollectionsAPIDistributedZkTest, HdfsRecoveryZkTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=HdfsRecoveryZkTest -Dtests.seed=7AC57E5DEEE16784 -Dtests.slow=true -Dtests.locale=uk-UA -Dtests.timezone=Europe/Tallinn -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | HdfsRecoveryZkTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
[junit4] > at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
[junit4] > at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
[junit4] > at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
[junit4] > at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)
[junit4] > at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)
[junit4] > at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([7AC57E5DEEE16784]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] Completed [526/662 (1!)] on J0 in 58.42s, 1 test, 1 failure <<< FAILURES!
[...truncated 62864 lines...]
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 556 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/556/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
4 tests failed.
FAILED: org.apache.solr.cloud.TestCloudPivotFacet.test
Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:53510 within 30000 ms
Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:53510 within 30000 ms
at __randomizedtesting.SeedInfo.seed([A18C22F895F66719:29D81D223B0A0AE1]:0)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:182)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:111)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:98)
at org.apache.solr.cloud.AbstractDistribZkTestBase.printLayout(AbstractDistribZkTestBase.java:322)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.distribTearDown(AbstractFullDistribZkTestBase.java:1500)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:969)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:53510 within 30000 ms
at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:174)
... 37 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestCloudPivotFacet
Error Message:
85 threads leaked from SUITE scope at org.apache.solr.cloud.TestCloudPivotFacet: 1) Thread[id=19116, name=qtp1245205965-19116, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=19114, name=qtp1245205965-19114, state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=19148, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-EventThread, state=WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 4) Thread[id=19184, name=qtp1188996271-19184, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=19213, name=qtp532655107-19213-acceptor-0@409d294f-ServerConnector@2534a649{HTTP/1.1,[http/1.1]}{127.0.0.1:64812}, state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=19256, name=org.eclipse.jetty.server.session.HashSessionManager@6de162f2Timer, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=19247, name=qtp85884834-19247, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=19221, name=org.eclipse.jetty.server.session.HashSessionManager@340b7cfeTimer, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=19123, name=org.eclipse.jetty.server.session.HashSessionManager@6dfe6310Timer, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=19147, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-SendThread(127.0.0.1:53510), state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 11) Thread[id=19160, name=qtp994564329-19160, state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 12) Thread[id=19171, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-SendThread(127.0.0.1:53510), state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 13) Thread[id=19218, name=qtp532655107-19218, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 14) Thread[id=19143, name=searcherExecutor-8595-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 15) Thread[id=19235, name=Thread-5041, state=WAITING, group=TGRP-TestCloudPivotFacet] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1274) 16) Thread[id=19220, name=Scheduler-1390796063, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 17) Thread[id=19259, name=Scheduler-699894630, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 18) Thread[id=19132, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-SendThread(127.0.0.1:53510), state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 19) Thread[id=19176, name=Thread-5023, state=WAITING, group=TGRP-TestCloudPivotFacet] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1274) 20) Thread[id=19187, name=qtp1188996271-19187, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 21) Thread[id=19115, name=qtp1245205965-19115-acceptor-0@335f7ed6-ServerConnector@5ffca595{HTTP/1.1,[http/1.1]}{127.0.0.1:52506}, state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 22) Thread[id=19136, name=OverseerCollectionConfigSetProcessor-97112934817595396-127.0.0.1:52506_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:343) at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:227) at java.lang.Thread.run(Thread.java:745) 23) Thread[id=19201, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-EventThread, state=WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 24) Thread[id=19214, name=qtp532655107-19214, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 25) Thread[id=19119, name=qtp1245205965-19119, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 26) Thread[id=19249, name=qtp85884834-19249, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 27) Thread[id=19137, name=OverseerHdfsCoreFailoverThread-97112934817595396-127.0.0.1:52506_-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139) at java.lang.Thread.run(Thread.java:745) 28) Thread[id=19217, name=qtp532655107-19217, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 29) Thread[id=19252, name=qtp85884834-19252, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 30) Thread[id=19156, name=qtp994564329-19156, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 31) Thread[id=19219, name=qtp532655107-19219, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 32) Thread[id=19107, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120) 33) Thread[id=19239, name=searcherExecutor-8628-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 34) Thread[id=19127, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:187) at java.lang.Thread.run(Thread.java:745) 35) Thread[id=19248, name=qtp85884834-19248-acceptor-0@35b57699-ServerConnector@4cbf511c{HTTP/1.1,[http/1.1]}{127.0.0.1:53510}, state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 36) Thread[id=19216, name=qtp532655107-19216, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 37) Thread[id=19153, name=qtp994564329-19153, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 38) Thread[id=19195, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:187) at java.lang.Thread.run(Thread.java:745) 39) Thread[id=19210, name=searcherExecutor-8617-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 40) Thread[id=19253, name=qtp85884834-19253, state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 41) Thread[id=19189, name=qtp1188996271-19189, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 42) Thread[id=19159, name=qtp994564329-19159, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 43) Thread[id=19200, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-SendThread(127.0.0.1:53510), state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 44) Thread[id=19254, name=qtp85884834-19254, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 45) Thread[id=19196, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:187) at java.lang.Thread.run(Thread.java:745) 46) Thread[id=19155, name=qtp994564329-19155, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 47) Thread[id=19118, name=qtp1245205965-19118, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 48) Thread[id=19103, name=Thread-5005, state=WAITING, group=TGRP-TestCloudPivotFacet] at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1249) at java.lang.Thread.join(Thread.java:1323) at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302) at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309) at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490) 49) Thread[id=19166, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:187) at java.lang.Thread.run(Thread.java:745) 50) Thread[id=19135, name=OverseerStateUpdate-97112934817595396-127.0.0.1:52506_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:138) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:179) at java.lang.Thread.run(Thread.java:745) 51) Thread[id=19225, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:187) at java.lang.Thread.run(Thread.java:745) 52) Thread[id=19246, name=Scheduler-1184804792, state=WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 53) Thread[id=19212, name=qtp532655107-19212, state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 54) Thread[id=19180, name=searcherExecutor-8606-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 55) Thread[id=19182, name=qtp1188996271-19182, state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 56) Thread[id=19206, name=Thread-5032, state=WAITING, group=TGRP-TestCloudPivotFacet] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1274) 57) Thread[id=19186, name=qtp1188996271-19186, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 58) Thread[id=19172, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-EventThread, state=WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 59) Thread[id=19255, name=Scheduler-2056197254, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 60) Thread[id=19190, name=Scheduler-821847665, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 61) Thread[id=19167, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:187) at java.lang.Thread.run(Thread.java:745) 62) Thread[id=19158, name=qtp994564329-19158, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 63) Thread[id=19128, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:187) at java.lang.Thread.run(Thread.java:745) 64) Thread[id=19117, name=qtp1245205965-19117, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 65) Thread[id=19226, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:187) at java.lang.Thread.run(Thread.java:745) 66) Thread[id=19105, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at java.lang.Object.wait(Native Method) at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146) 67) Thread[id=19133, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-EventThread, state=WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 68) Thread[id=19157, name=qtp994564329-19157, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 69) Thread[id=19139, name=Thread-5014, state=WAITING, group=TGRP-TestCloudPivotFacet] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1274) 70) Thread[id=19183, name=qtp1188996271-19183-acceptor-0@2f30661f-ServerConnector@24489b15{HTTP/1.1,[http/1.1]}{127.0.0.1:58529}, state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 71) Thread[id=19154, name=qtp994564329-19154-acceptor-0@45cf7183-ServerConnector@326c47e9{HTTP/1.1,[http/1.1]}{127.0.0.1:45140}, state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:745) 72) Thread[id=19188, name=qtp1188996271-19188, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 73) Thread[id=19104, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178) at java.lang.Thread.run(Thread.java:745) 74) Thread[id=19162, name=org.eclipse.jetty.server.session.HashSessionManager@2274e0f1Timer, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 75) Thread[id=19161, name=Scheduler-701754175, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 76) Thread[id=19231, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-EventThread, state=WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 77) Thread[id=19251, name=qtp85884834-19251, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 78) Thread[id=19215, name=qtp532655107-19215, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 79) Thread[id=19185, name=qtp1188996271-19185, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 80) Thread[id=19250, name=qtp85884834-19250, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 81) Thread[id=19121, name=qtp1245205965-19121, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 82) Thread[id=19230, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-SendThread(127.0.0.1:53510), state=RUNNABLE, group=TGRP-TestCloudPivotFacet] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) 83) Thread[id=19122, name=Scheduler-1697959167, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 84) Thread[id=19120, name=qtp1245205965-19120, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:745) 85) Thread[id=19191, name=org.eclipse.jetty.server.session.HashSessionManager@322d5441Timer, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 85 threads leaked from SUITE scope at org.apache.solr.cloud.TestCloudPivotFacet:
1) Thread[id=19116, name=qtp1245205965-19116, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
2) Thread[id=19114, name=qtp1245205965-19114, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
3) Thread[id=19148, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-EventThread, state=WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
4) Thread[id=19184, name=qtp1188996271-19184, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
5) Thread[id=19213, name=qtp532655107-19213-acceptor-0@409d294f-ServerConnector@2534a649{HTTP/1.1,[http/1.1]}{127.0.0.1:64812}, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
6) Thread[id=19256, name=org.eclipse.jetty.server.session.HashSessionManager@6de162f2Timer, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
7) Thread[id=19247, name=qtp85884834-19247, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
8) Thread[id=19221, name=org.eclipse.jetty.server.session.HashSessionManager@340b7cfeTimer, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
9) Thread[id=19123, name=org.eclipse.jetty.server.session.HashSessionManager@6dfe6310Timer, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
10) Thread[id=19147, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-SendThread(127.0.0.1:53510), state=RUNNABLE, group=TGRP-TestCloudPivotFacet]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
11) Thread[id=19160, name=qtp994564329-19160, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
12) Thread[id=19171, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-SendThread(127.0.0.1:53510), state=RUNNABLE, group=TGRP-TestCloudPivotFacet]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
13) Thread[id=19218, name=qtp532655107-19218, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
14) Thread[id=19143, name=searcherExecutor-8595-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
15) Thread[id=19235, name=Thread-5041, state=WAITING, group=TGRP-TestCloudPivotFacet]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1274)
16) Thread[id=19220, name=Scheduler-1390796063, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
17) Thread[id=19259, name=Scheduler-699894630, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
18) Thread[id=19132, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-SendThread(127.0.0.1:53510), state=RUNNABLE, group=TGRP-TestCloudPivotFacet]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
19) Thread[id=19176, name=Thread-5023, state=WAITING, group=TGRP-TestCloudPivotFacet]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1274)
20) Thread[id=19187, name=qtp1188996271-19187, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
21) Thread[id=19115, name=qtp1245205965-19115-acceptor-0@335f7ed6-ServerConnector@5ffca595{HTTP/1.1,[http/1.1]}{127.0.0.1:52506}, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
22) Thread[id=19136, name=OverseerCollectionConfigSetProcessor-97112934817595396-127.0.0.1:52506_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:343)
at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)
at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:227)
at java.lang.Thread.run(Thread.java:745)
23) Thread[id=19201, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-EventThread, state=WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
24) Thread[id=19214, name=qtp532655107-19214, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
25) Thread[id=19119, name=qtp1245205965-19119, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
26) Thread[id=19249, name=qtp85884834-19249, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
27) Thread[id=19137, name=OverseerHdfsCoreFailoverThread-97112934817595396-127.0.0.1:52506_-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
at java.lang.Thread.run(Thread.java:745)
28) Thread[id=19217, name=qtp532655107-19217, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
29) Thread[id=19252, name=qtp85884834-19252, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:745)
30) Thread[id=19156, name=qtp994564329-19156, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueu
[...truncated too long message...]
r.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 20) Thread[id=19132, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-SendThread(127.0.0.1:53510), state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
[junit4] > 21) Thread[id=19176, name=Thread-5023, state=WAITING, group=TGRP-TestCloudPivotFacet]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1274)
[junit4] > 22) Thread[id=19139, name=Thread-5014, state=WAITING, group=TGRP-TestCloudPivotFacet]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1274)
[junit4] > 23) Thread[id=19162, name=org.eclipse.jetty.server.session.HashSessionManager@2274e0f1Timer, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 24) Thread[id=19239, name=searcherExecutor-8628-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 25) Thread[id=19161, name=Scheduler-701754175, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 26) Thread[id=19230, name=TEST-TestCloudPivotFacet.test-seed#[A18C22F895F66719]-SendThread(127.0.0.1:53510), state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
[junit4] > 27) Thread[id=19210, name=searcherExecutor-8617-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 28) Thread[id=19122, name=Scheduler-1697959167, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 29) Thread[id=19253, name=qtp85884834-19253, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 30) Thread[id=19191, name=org.eclipse.jetty.server.session.HashSessionManager@322d5441Timer, state=TIMED_WAITING, group=TGRP-TestCloudPivotFacet]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([A18C22F895F66719]:0)
[junit4] Completed [507/662 (2!)] on J0 in 125.72s, 1 test, 3 errors <<< FAILURES!
[...truncated 62914 lines...]