You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/01/09 22:39:35 UTC
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 608 -
Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/608/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.testDelegationTokenCancelFail
Error Message:
expected:<200> but was:<404>
Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
at __randomizedtesting.SeedInfo.seed([F4BBD27FF6AE0C83:9C04E75526341E6F]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.cancelDelegationToken(TestDelegationWithHadoopAuth.java:128)
at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.testDelegationTokenCancelFail(TestDelegationWithHadoopAuth.java:280)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 10979 lines...]
[junit4] Suite: org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_F4BBD27FF6AE0C83-001/init-core-data-001
[junit4] 2> 275036 INFO (SUITE-TestDelegationWithHadoopAuth-seed#[F4BBD27FF6AE0C83]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 275036 INFO (SUITE-TestDelegationWithHadoopAuth-seed#[F4BBD27FF6AE0C83]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_F4BBD27FF6AE0C83-001/tempDir-001
[junit4] 2> 275037 INFO (SUITE-TestDelegationWithHadoopAuth-seed#[F4BBD27FF6AE0C83]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 275037 INFO (Thread-2806) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 275037 INFO (Thread-2806) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 275137 INFO (SUITE-TestDelegationWithHadoopAuth-seed#[F4BBD27FF6AE0C83]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:48780
[junit4] 2> 275151 INFO (jetty-launcher-341-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 275151 INFO (jetty-launcher-341-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 275153 INFO (jetty-launcher-341-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5c96ac7a{/solr,null,AVAILABLE}
[junit4] 2> 275153 INFO (jetty-launcher-341-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1488c8e6{/solr,null,AVAILABLE}
[junit4] 2> 275163 INFO (jetty-launcher-341-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@52eb8e9d{HTTP/1.1,[http/1.1]}{127.0.0.1:34105}
[junit4] 2> 275163 INFO (jetty-launcher-341-thread-1) [ ] o.e.j.s.Server Started @278245ms
[junit4] 2> 275163 INFO (jetty-launcher-341-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34105}
[junit4] 2> 275164 ERROR (jetty-launcher-341-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 275164 INFO (jetty-launcher-341-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 275164 INFO (jetty-launcher-341-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 275165 INFO (jetty-launcher-341-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 275165 INFO (jetty-launcher-341-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-09T21:38:25.975Z
[junit4] 2> 275165 INFO (jetty-launcher-341-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4a31b941{HTTP/1.1,[http/1.1]}{127.0.0.1:54275}
[junit4] 2> 275166 INFO (jetty-launcher-341-thread-2) [ ] o.e.j.s.Server Started @278247ms
[junit4] 2> 275166 INFO (jetty-launcher-341-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54275}
[junit4] 2> 275166 ERROR (jetty-launcher-341-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 275166 INFO (jetty-launcher-341-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 275166 INFO (jetty-launcher-341-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 275166 INFO (jetty-launcher-341-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 275166 INFO (jetty-launcher-341-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-09T21:38:25.976Z
[junit4] 2> 275171 INFO (jetty-launcher-341-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 275173 INFO (jetty-launcher-341-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 275181 INFO (jetty-launcher-341-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 275181 INFO (jetty-launcher-341-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 275182 INFO (jetty-launcher-341-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48780/solr
[junit4] 2> 275183 INFO (jetty-launcher-341-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48780/solr
[junit4] 2> 275226 INFO (jetty-launcher-341-thread-1) [n:127.0.0.1:34105_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34105_solr
[junit4] 2> 275227 INFO (jetty-launcher-341-thread-2) [n:127.0.0.1:54275_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54275_solr
[junit4] 2> 275228 INFO (jetty-launcher-341-thread-1) [n:127.0.0.1:34105_solr ] o.a.s.c.Overseer Overseer (id=97255286757720070-127.0.0.1:34105_solr-n_0000000000) starting
[junit4] 2> 275303 INFO (zkCallback-352-thread-1-processing-n:127.0.0.1:34105_solr) [n:127.0.0.1:34105_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 275303 INFO (zkCallback-351-thread-1-processing-n:127.0.0.1:54275_solr) [n:127.0.0.1:54275_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 275385 INFO (jetty-launcher-341-thread-1) [n:127.0.0.1:34105_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34105_solr
[junit4] 2> 275388 INFO (jetty-launcher-341-thread-2) [n:127.0.0.1:54275_solr ] o.a.s.c.CoreContainer Authentication plugin class obtained from ZK: org.apache.solr.security.HadoopAuthPlugin
[junit4] 2> 275393 INFO (zkCallback-351-thread-1-processing-n:127.0.0.1:54275_solr) [n:127.0.0.1:54275_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 275394 INFO (OverseerStateUpdate-97255286757720070-127.0.0.1:34105_solr-n_0000000000) [n:127.0.0.1:34105_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 275401 INFO (jetty-launcher-341-thread-2) [n:127.0.0.1:54275_solr ] o.a.s.s.HadoopAuthPlugin Params: {delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, type=simple, zk-dt-secret-manager.enable=true, signer.secret.provider=zookeeper}
[junit4] 2> 275540 INFO (jetty-launcher-341-thread-1) [n:127.0.0.1:34105_solr ] o.a.s.c.CoreContainer Authentication plugin class obtained from ZK: org.apache.solr.security.HadoopAuthPlugin
[junit4] 2> 275540 INFO (jetty-launcher-341-thread-1) [n:127.0.0.1:34105_solr ] o.a.s.s.HadoopAuthPlugin Params: {delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, type=simple, zk-dt-secret-manager.enable=true, signer.secret.provider=zookeeper}
[junit4] 2> 275608 INFO (jetty-launcher-341-thread-2) [n:127.0.0.1:54275_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 275608 INFO (jetty-launcher-341-thread-1) [n:127.0.0.1:34105_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 275688 INFO (jetty-launcher-341-thread-2-EventThread) [n:127.0.0.1:54275_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 275690 INFO (jetty-launcher-341-thread-1-EventThread) [n:127.0.0.1:34105_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 275926 INFO (jetty-launcher-341-thread-1) [n:127.0.0.1:34105_solr ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
[junit4] 2> 275926 INFO (jetty-launcher-341-thread-1) [n:127.0.0.1:34105_solr ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.PKIAuthenticationPlugin$HttpHeaderClientConfigurer@49fa6f98
[junit4] 2> 275926 INFO (jetty-launcher-341-thread-1) [n:127.0.0.1:34105_solr ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.PKIAuthenticationPlugin$HttpHeaderClientConfigurer@49fa6f98
[junit4] 2> 275929 INFO (jetty-launcher-341-thread-1) [n:127.0.0.1:34105_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_F4BBD27FF6AE0C83-001/tempDir-001/node1/.
[junit4] 2> 275946 INFO (jetty-launcher-341-thread-2) [n:127.0.0.1:54275_solr ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
[junit4] 2> 275946 INFO (jetty-launcher-341-thread-2) [n:127.0.0.1:54275_solr ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.PKIAuthenticationPlugin$HttpHeaderClientConfigurer@6b2c5b60
[junit4] 2> 275947 INFO (jetty-launcher-341-thread-2) [n:127.0.0.1:54275_solr ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.PKIAuthenticationPlugin$HttpHeaderClientConfigurer@6b2c5b60
[junit4] 2> 275950 INFO (jetty-launcher-341-thread-2) [n:127.0.0.1:54275_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_F4BBD27FF6AE0C83-001/tempDir-001/node2/.
[junit4] 2> 276063 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancelFail-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
[junit4] 2> 276201 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancelFail-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestDelegationWithHadoopAuth -Dtests.method=testDelegationTokenCancelFail -Dtests.seed=F4BBD27FF6AE0C83 -Dtests.slow=true -Dtests.locale=de -Dtests.timezone=America/Anchorage -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 0.17s J0 | TestDelegationWithHadoopAuth.testDelegationTokenCancelFail <<<
[junit4] > Throwable #1: java.lang.AssertionError: expected:<200> but was:<404>
[junit4] > at __randomizedtesting.SeedInfo.seed([F4BBD27FF6AE0C83:9C04E75526341E6F]:0)
[junit4] > at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.cancelDelegationToken(TestDelegationWithHadoopAuth.java:128)
[junit4] > at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.testDelegationTokenCancelFail(TestDelegationWithHadoopAuth.java:280)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 276234 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
[junit4] 2> 276307 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 276309 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48780/solr ready
[junit4] 2> 276320 INFO (qtp1758877489-4418) [n:127.0.0.1:54275_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwhmigFZqTeMZgIDFFSn0tNRjSbWk58SfYFvu-J2dYTUB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 276320 INFO (qtp1758877489-4418) [n:127.0.0.1:54275_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwhmigFZqTeMZgIDFFSn0tNRjSbWk58SfYFvu-J2dYTUB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 276329 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 276330 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48780/solr ready
[junit4] 2> 276351 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 276361 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 276371 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48780/solr ready
[junit4] 2> 276375 INFO (qtp1758877489-4420) [n:127.0.0.1:54275_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwhmigFZqTeMZgIDFFSn0tNRjSbWk58SfYFvu-J2dYTUB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 276375 INFO (qtp1758877489-4420) [n:127.0.0.1:54275_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwhmigFZqTeMZgIDFFSn0tNRjSbWk58SfYFvu-J2dYTUB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 276385 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 276388 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48780/solr ready
[junit4] 2> 276391 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 276504 INFO (qtp1758758205-4412) [n:127.0.0.1:34105_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwktigFZqTeNLQMDFCxYiMlOaSrJ4dfthFLuSZeMJW6TB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 276504 INFO (qtp1758758205-4412) [n:127.0.0.1:34105_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwktigFZqTeNLQMDFCxYiMlOaSrJ4dfthFLuSZeMJW6TB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 276509 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 276511 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48780/solr ready
[junit4] 2> 276514 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 276520 INFO (qtp1758877489-4420) [n:127.0.0.1:54275_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwktigFZqTeNLQMDFCxYiMlOaSrJ4dfthFLuSZeMJW6TB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 276520 INFO (qtp1758877489-4420) [n:127.0.0.1:54275_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwktigFZqTeNLQMDFCxYiMlOaSrJ4dfthFLuSZeMJW6TB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 276522 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 276540 INFO (qtp1758758205-4407) [n:127.0.0.1:34105_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwmHigFZqTeNhwQDFNYYFjHL_vub3e7__rx9SArvRhMBB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 276540 INFO (qtp1758758205-4407) [n:127.0.0.1:34105_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwmHigFZqTeNhwQDFNYYFjHL_vub3e7__rx9SArvRhMBB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 276543 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 276549 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 276552 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48780/solr ready
[junit4] 2> 276554 INFO (qtp1758877489-4424) [n:127.0.0.1:54275_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwmHigFZqTeNhwQDFNYYFjHL_vub3e7__rx9SArvRhMBB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 276554 INFO (qtp1758877489-4424) [n:127.0.0.1:54275_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwmHigFZqTeNhwQDFNYYFjHL_vub3e7__rx9SArvRhMBB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 276561 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 276562 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48780/solr ready
[junit4] 2> 276565 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 276603 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 276605 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48780/solr ready
[junit4] 2> 276607 INFO (qtp1758877489-4423) [n:127.0.0.1:54275_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwnAigFZqTeNwAUDFCKQr0XNSvT0Wb78ILOg-6VdUyhnB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 276608 INFO (qtp1758877489-4423) [n:127.0.0.1:54275_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwnAigFZqTeNwAUDFCKQr0XNSvT0Wb78ILOg-6VdUyhnB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 276614 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 276616 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48780/solr ready
[junit4] 2> 276618 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 276654 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 276657 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48780/solr ready
[junit4] 2> 276661 INFO (qtp1758877489-4421) [n:127.0.0.1:54275_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwnAigFZqTeNwAUDFCKQr0XNSvT0Wb78ILOg-6VdUyhnB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 276661 INFO (qtp1758877489-4421) [n:127.0.0.1:54275_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwnAigFZqTeNwAUDFCKQr0XNSvT0Wb78ILOg-6VdUyhnB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 276667 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 276667 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
[junit4] 2> 276815 INFO (TEST-TestDelegationWithHadoopAuth.testZNodePaths-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
[junit4] 2> 276832 INFO (TEST-TestDelegationWithHadoopAuth.testZNodePaths-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
[junit4] 2> 276855 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
[junit4] 2> 276868 INFO (qtp1758758205-4408) [n:127.0.0.1:34105_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwrUigFZqTeO1AcDFLuu4klJNZLzk3ka-F_pyTqedSdxB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 276868 INFO (qtp1758758205-4408) [n:127.0.0.1:34105_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwrUigFZqTeO1AcDFLuu4klJNZLzk3ka-F_pyTqedSdxB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 276873 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 276874 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48780/solr ready
[junit4] 2> 276877 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 276883 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 276884 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48780/solr ready
[junit4] 2> 276887 INFO (qtp1758877489-4423) [n:127.0.0.1:54275_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwrUigFZqTeO1AcDFLuu4klJNZLzk3ka-F_pyTqedSdxB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 276887 INFO (qtp1758877489-4423) [n:127.0.0.1:54275_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwrUigFZqTeO1AcDFLuu4klJNZLzk3ka-F_pyTqedSdxB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 276892 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 276892 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
[junit4] 2> 276924 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
[junit4] 2> 276942 ERROR (qtp1758758205-4413) [ ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_8]
[junit4] 2> 276952 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 276954 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48780/solr ready
[junit4] 2> 276956 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 276963 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 276964 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48780/solr ready
[junit4] 2> 276967 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 277012 ERROR (qtp1758877489-4418) [n:127.0.0.1:54275_solr ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_9]
[junit4] 2> 277023 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 277028 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 277030 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48780/solr ready
[junit4] 2> 277039 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 277041 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
[junit4] 2> 277083 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
[junit4] 2> 277102 WARN (qtp1758758205-4410) [ ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: bar tries to renew a token with renewer foo
[junit4] 2> 277102 WARN (qtp1758758205-4410) [ ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
[junit4] 2> 277110 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 277112 WARN (qtp1758877489-4423) [n:127.0.0.1:54275_solr ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: bar tries to renew a token with renewer foo
[junit4] 2> 277113 WARN (qtp1758877489-4423) [n:127.0.0.1:54275_solr ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
[junit4] 2> 277116 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 277128 WARN (qtp1758758205-4407) [n:127.0.0.1:34105_solr ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
[junit4] 2> 277128 WARN (qtp1758758205-4407) [n:127.0.0.1:34105_solr ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
[junit4] 2> 277129 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 277134 WARN (qtp1758877489-4421) [n:127.0.0.1:54275_solr ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
[junit4] 2> 277134 WARN (qtp1758877489-4421) [n:127.0.0.1:54275_solr ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
[junit4] 2> 277134 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 277134 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
[junit4] 2> 277158 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenSolrClient-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
[junit4] 2> 277189 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationTokenSolrClient-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=Do, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Do, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 277192 INFO (qtp1758758205-4407) [n:127.0.0.1:34105_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwwDigFZqTeQAwwDFHPF_FymSG4XEQSRum4NWpHu4WXSB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 277192 INFO (qtp1758758205-4407) [n:127.0.0.1:34105_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVmFKwwDigFZqTeQAwwDFHPF_FymSG4XEQSRum4NWpHu4WXSB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 277192 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationTokenSolrClient-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
[junit4] 2> 277214 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationOtherOp-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
[junit4] 2> 277217 INFO (qtp1758758205-4413) [n:127.0.0.1:34105_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&user.name=foo&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 277217 INFO (qtp1758758205-4413) [n:127.0.0.1:34105_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&user.name=foo&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 277217 WARN (TEST-TestDelegationWithHadoopAuth.testDelegationOtherOp-seed#[F4BBD27FF6AE0C83]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=foo&p=foo&t=simple-dt&e=1484033908026&s=mDlOUxKiAAvz9zcJPiXXnw5sGjY="; Domain=127.0.0.1; Expires=Di, 10-Jan-2017 07:38:28 GMT; HttpOnly". Invalid 'expires' attribute: Di, 10-Jan-2017 07:38:28 GMT
[junit4] 2> 277217 INFO (TEST-TestDelegationWithHadoopAuth.testDelegationOtherOp-seed#[F4BBD27FF6AE0C83]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
[junit4] 2> 277219 INFO (jetty-launcher-341-thread-2) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@52eb8e9d{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 277220 INFO (jetty-launcher-341-thread-2) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1471530408
[junit4] 2> 277220 INFO (jetty-launcher-341-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@4a31b941{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 277220 INFO (jetty-launcher-341-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=462619024
[junit4] 2> 277230 INFO (jetty-launcher-341-thread-2) [ ] o.a.s.c.Overseer Overseer (id=97255286757720070-127.0.0.1:34105_solr-n_0000000000) closing
[junit4] 2> 277230 INFO (OverseerStateUpdate-97255286757720070-127.0.0.1:34105_solr-n_0000000000) [n:127.0.0.1:34105_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:34105_solr
[junit4] 2> 277233 INFO (jetty-launcher-341-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 277234 INFO (jetty-launcher-341-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 277234 ERROR (Thread[Thread-2825,5,TGRP-TestDelegationWithHadoopAuth]) [n:127.0.0.1:54275_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 277235 ERROR (Thread[Thread-2823,5,TGRP-TestDelegationWithHadoopAuth]) [n:127.0.0.1:34105_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 277250 INFO (jetty-launcher-341-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1488c8e6{/solr,null,UNAVAILABLE}
[junit4] 2> 277253 INFO (jetty-launcher-341-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5c96ac7a{/solr,null,UNAVAILABLE}
[junit4] 2> 277259 INFO (SUITE-TestDelegationWithHadoopAuth-seed#[F4BBD27FF6AE0C83]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:48780 48780
[junit4] 2> 277326 INFO (Thread-2806) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:48780 48780
[junit4] 2> 277332 WARN (Thread-2806) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 17 /solr/aliases.json
[junit4] 2> 17 /solr/clusterprops.json
[junit4] 2> 2 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 17 /solr/clusterstate.json
[junit4] 2> 3 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_12
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_11
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 17 /solr/live_nodes
[junit4] 2> 17 /solr/collections
[junit4] 2> 3 /solr/overseer/queue
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
[junit4] 2>
[junit4] 2> 277336 INFO (SUITE-TestDelegationWithHadoopAuth-seed#[F4BBD27FF6AE0C83]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_F4BBD27FF6AE0C83-001
[junit4] 2> Jan 09, 2017 9:38:28 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 3 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=930, maxMBSortInHeap=6.341331829323289, sim=RandomSimilarity(queryNorm=true,coord=yes): {}, locale=de, timezone=America/Anchorage
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=1,free=179842360,total=343932928
[junit4] 2> NOTE: All tests run in this JVM: [PreAnalyzedFieldManagedSchemaCloudTest, DeleteStatusTest, SimpleFacetsTest, CdcrUpdateLogTest, SystemInfoHandlerTest, PeerSyncWithIndexFingerprintCachingTest, BlockJoinFacetDistribTest, TestFieldCacheVsDocValues, OverseerRolesTest, ClassificationUpdateProcessorFactoryTest, TestStressCloudBlindAtomicUpdates, DateRangeFieldTest, TestSchemaResource, ActionThrottleTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestMacroExpander, SoftAutoCommitTest, HdfsRecoveryZkTest, UnloadDistributedZkTest, DocumentBuilderTest, TestManagedSynonymFilterFactory, TestScoreJoinQPNoScore, HdfsWriteToMultipleCollectionsTest, StatelessScriptUpdateProcessorFactoryTest, CheckHdfsIndexTest, TestDelegationWithHadoopAuth]
[junit4] Completed [77/679 (1!)] on J0 in 3.33s, 8 tests, 1 failure <<< FAILURES!
[...truncated 64420 lines...]
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 613 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/613/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC
1 tests failed.
FAILED: org.apache.solr.cloud.TestZkChroot.testNoBootstrapConf
Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:39947 within 30000 ms
Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:39947 within 30000 ms
at __randomizedtesting.SeedInfo.seed([11E173D93FE82664:7102815FC0B042E1]:0)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:182)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:111)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:98)
at org.apache.solr.cloud.TestZkChroot.testNoBootstrapConf(TestZkChroot.java:106)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:39947 within 30000 ms
at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:174)
... 43 more
Build Log:
[...truncated 10822 lines...]
[junit4] Suite: org.apache.solr.cloud.TestZkChroot
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/init-core-data-001
[junit4] 2> 253512 INFO (SUITE-TestZkChroot-seed#[11E173D93FE82664]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 253565 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testWithUploadDir
[junit4] 2> 253565 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 253565 INFO (Thread-539) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 253565 INFO (Thread-539) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 253669 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [ ] o.a.s.c.ZkTestServer start zk server on port:46584
[junit4] 2> 253755 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/solr.xml
[junit4] 2> 253762 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
[junit4] 2> 253763 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46584/foo/bar3
[junit4] 2> 253824 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 192.168.57.18:8983_solr
[junit4] 2> 253825 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.Overseer Overseer (id=97267451825029122-192.168.57.18:8983_solr-n_0000000000) starting
[junit4] 2> 253838 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/192.168.57.18:8983_solr
[junit4] 2> 253841 INFO (zkCallback-234-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 254196 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001
[junit4] 2> 254196 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 254199 INFO (OverseerStateUpdate-97267451825029122-192.168.57.18:8983_solr-n_0000000000) [n:192.168.57.18:8983_solr ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 255202 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.CreateCollectionCmd Setting config for collection:collection1 to testWithUploadDir
[junit4] 2> 255214 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.RequestParams conf resource params.json loaded . version : 0
[junit4] 2> 255215 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.RequestParams request params refreshed to version 0
[junit4] 2> 255218 WARN (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../contrib/extraction/lib filtered by .*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../contrib/extraction/lib
[junit4] 2> 255218 WARN (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../dist filtered by solr-cell-\d.*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../dist
[junit4] 2> 255218 WARN (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../contrib/clustering/lib filtered by .*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../contrib/clustering/lib
[junit4] 2> 255218 WARN (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../dist filtered by solr-clustering-\d.*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../dist
[junit4] 2> 255219 WARN (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../contrib/langid/lib filtered by .*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../contrib/langid/lib
[junit4] 2> 255219 WARN (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../dist filtered by solr-langid-\d.*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../dist
[junit4] 2> 255219 WARN (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../contrib/ltr/lib filtered by .*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../contrib/ltr/lib
[junit4] 2> 255219 WARN (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../dist filtered by solr-ltr-\d.*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../dist
[junit4] 2> 255219 WARN (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../contrib/velocity/lib filtered by .*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../contrib/velocity/lib
[junit4] 2> 255220 WARN (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.SolrConfig Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../dist filtered by solr-velocity-\d.*\.jar to classpath: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/../../../../dist
[junit4] 2> 255233 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.SolrIndexConfig IndexWriter infoStream solr logging is enabled
[junit4] 2> 255234 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 255281 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=example
[junit4] 2> 255742 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.RestManager Registered ManagedResource impl org.apache.solr.rest.schema.analysis.ManagedWordSetResource for path /schema/analysis/stopwords/english
[junit4] 2> 255742 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.RestManager Registered ManagedResource impl org.apache.solr.rest.schema.analysis.ManagedSynonymFilterFactory$SynonymManager for path /schema/analysis/synonyms/english
[junit4] 2> 255751 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema example/1.6 with uniqueid field id
[junit4] 2> 255751 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.RestManager Added observer of type org.apache.solr.rest.schema.analysis.ManagedStopFilterFactory to existing ManagedResource /schema/analysis/stopwords/english
[junit4] 2> 255751 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.RestManager Added observer of type org.apache.solr.rest.schema.analysis.ManagedSynonymFilterFactory to existing ManagedResource /schema/analysis/synonyms/english
[junit4] 2> 255819 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 255819 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001/tempDir-001/collection1/data/]
[junit4] 2> 255819 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@86bdc80
[junit4] 2> 255829 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: init: current segments file is "segments"; deletionPolicy=org.apache.solr.core.IndexDeletionPolicyWrapper@63b121a5
[junit4] 2> 255829 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: delete []
[junit4] 2> 255829 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: now checkpoint "" [0 segments ; isCommit = false]
[junit4] 2> 255829 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: delete []
[junit4] 2> 255829 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: 0 msec to checkpoint
[junit4] 2> 255829 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: init: create=true
[junit4] 2> 255830 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]:
[junit4] 2> dir=MetricsDirectory(MockDirectoryWrapper(RAMDirectory@2d1a887f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2d9fbdfb))
[junit4] 2> index=
[junit4] 2> version=6.4.0
[junit4] 2> analyzer=org.apache.solr.update.SolrIndexConfig$DelayedSchemaAnalyzer
[junit4] 2> ramBufferSizeMB=100.0
[junit4] 2> maxBufferedDocs=-1
[junit4] 2> maxBufferedDeleteTerms=-1
[junit4] 2> mergedSegmentWarmer=null
[junit4] 2> delPolicy=org.apache.solr.core.IndexDeletionPolicyWrapper
[junit4] 2> commit=null
[junit4] 2> openMode=CREATE
[junit4] 2> similarity=org.apache.solr.search.similarities.SchemaSimilarityFactory$SchemaSimilarity
[junit4] 2> mergeScheduler=ConcurrentMergeScheduler: maxThreadCount=-1, maxMergeCount=-1, ioThrottle=true
[junit4] 2> codec=Lucene62
[junit4] 2> infoStream=org.apache.solr.update.LoggingInfoStream
[junit4] 2> mergePolicy=[TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=5120.0, floorSegmentMB=2.0, forceMergeDeletesPctAllowed=10.0, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1
[junit4] 2> indexerThreadPool=org.apache.lucene.index.DocumentsWriterPerThreadPool@214afca2
[junit4] 2> readerPooling=false
[junit4] 2> perThreadHardLimitMB=1945
[junit4] 2> useCompoundFile=false
[junit4] 2> commitOnClose=true
[junit4] 2> indexSort=null
[junit4] 2> writer=org.apache.solr.update.SolrIndexWriter@586ad184
[junit4] 2>
[junit4] 2> 255830 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: MMapDirectory.UNMAP_SUPPORTED=true
[junit4] 2> 255830 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: now flush at close
[junit4] 2> 255830 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: start flush: applyAllDeletes=true
[junit4] 2> 255830 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: index before flush
[junit4] 2> 255830 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [DW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: startFullFlush
[junit4] 2> 255830 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [DW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr finishFullFlush success=true
[junit4] 2> 255830 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: apply all deletes during flush
[junit4] 2> 255830 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: now apply all deletes for all segments maxDoc=0
[junit4] 2> 255830 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [BD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: prune sis=segments: minGen=9223372036854775807 packetCount=0
[junit4] 2> 255830 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [MS][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: initDynamicDefaults spins=true maxThreadCount=1 maxMergeCount=6
[junit4] 2> 255830 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [MS][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: updateMergeThreads ioThrottle=true targetMBPerSec=10240.0 MB/sec
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [MS][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: now merge
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [MS][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: index:
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [MS][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: no more merges pending; now return
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: waitForMerges
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: waitForMerges done
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: commit: start
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: commit: enter lock
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: commit: now prepare
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: prepareCommit: flush
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: index before flush
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [DW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: startFullFlush
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: apply all deletes during flush
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: now apply all deletes for all segments maxDoc=0
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [BD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: prune sis=segments: minGen=9223372036854775807 packetCount=0
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [DW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr finishFullFlush success=true
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: startCommit(): start
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: startCommit index= changeCount=2
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: startCommit: wrote pending segments file "pending_segments_1"
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: done all syncs: []
[junit4] 2> 255831 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: commit: pendingCommit != null
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: commit: done writing segments file "segments_1"
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: now checkpoint "" [0 segments ; isCommit = true]
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: 0 msec to checkpoint
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: delete []
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: commit: took 0.9 msec
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: commit: done
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: rollback
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: all running merges have aborted
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: rollback: done finish merges
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [DW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: abort
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [DW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: done abort success=true
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: rollback: infos=
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: now checkpoint "" [0 segments ; isCommit = false]
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: delete []
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: 0 msec to checkpoint
[junit4] 2> 255832 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: delete []
[junit4] 2> 255858 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.XSLTResponseWriter xsltCacheLifetimeSeconds=5
[junit4] 2> 256018 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 256018 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 256019 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 256020 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 256020 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: init: current segments file is "segments_1"; deletionPolicy=org.apache.solr.core.IndexDeletionPolicyWrapper@63b121a5
[junit4] 2> 256020 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: init: load commit "segments_1"
[junit4] 2> 256020 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: delete []
[junit4] 2> 256020 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: now checkpoint "" [0 segments ; isCommit = false]
[junit4] 2> 256020 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: delete []
[junit4] 2> 256021 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: 0 msec to checkpoint
[junit4] 2> 256021 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: init: create=false
[junit4] 2> 256021 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]:
[junit4] 2> dir=MetricsDirectory(MockDirectoryWrapper(RAMDirectory@2d1a887f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2d9fbdfb))
[junit4] 2> index=
[junit4] 2> version=6.4.0
[junit4] 2> analyzer=org.apache.solr.update.SolrIndexConfig$DelayedSchemaAnalyzer
[junit4] 2> ramBufferSizeMB=100.0
[junit4] 2> maxBufferedDocs=-1
[junit4] 2> maxBufferedDeleteTerms=-1
[junit4] 2> mergedSegmentWarmer=null
[junit4] 2> delPolicy=org.apache.solr.core.IndexDeletionPolicyWrapper
[junit4] 2> commit=null
[junit4] 2> openMode=APPEND
[junit4] 2> similarity=org.apache.solr.search.similarities.SchemaSimilarityFactory$SchemaSimilarity
[junit4] 2> mergeScheduler=ConcurrentMergeScheduler: maxThreadCount=-1, maxMergeCount=-1, ioThrottle=true
[junit4] 2> codec=Lucene62
[junit4] 2> infoStream=org.apache.solr.update.LoggingInfoStream
[junit4] 2> mergePolicy=[TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=5120.0, floorSegmentMB=2.0, forceMergeDeletesPctAllowed=10.0, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1
[junit4] 2> indexerThreadPool=org.apache.lucene.index.DocumentsWriterPerThreadPool@a3d8fee
[junit4] 2> readerPooling=false
[junit4] 2> perThreadHardLimitMB=1945
[junit4] 2> useCompoundFile=false
[junit4] 2> commitOnClose=true
[junit4] 2> indexSort=null
[junit4] 2> writer=org.apache.solr.update.SolrIndexWriter@74c4f063
[junit4] 2>
[junit4] 2> 256021 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: MMapDirectory.UNMAP_SUPPORTED=true
[junit4] 2> 256021 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: flush at getReader
[junit4] 2> 256021 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [DW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: startFullFlush
[junit4] 2> 256021 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: apply all deletes during flush
[junit4] 2> 256021 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: now apply all deletes for all segments maxDoc=0
[junit4] 2> 256021 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [BD][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: prune sis=segments_1: minGen=9223372036854775807 packetCount=0
[junit4] 2> 256021 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: incRefDeleter for NRT reader version=2 segments=
[junit4] 2> 256021 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: return reader version=2 reader=StandardDirectoryReader(segments_1:2:nrt)
[junit4] 2> 256021 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [DW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr finishFullFlush success=true
[junit4] 2> 256021 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr]: getReader took 0 msec
[junit4] 2> 256021 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6d8c5bb7[collection1] main]
[junit4] 2> 256023 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testWithUploadDir
[junit4] 2> 256024 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded LinkedHashMap at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testWithUploadDir
[junit4] 2> 256024 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.ManagedResource Loaded initArgs {} for /rest/managed
[junit4] 2> 256028 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded LinkedHashMap at path _schema_analysis_stopwords_english.json using ZooKeeperStorageIO:path=/configs/testWithUploadDir
[junit4] 2> 256028 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.ManagedResource Loaded initArgs {ignoreCase=true} for /schema/analysis/stopwords/english
[junit4] 2> 256029 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.s.a.ManagedWordSetResource Loaded 35 words for /schema/analysis/stopwords/english
[junit4] 2> 256029 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.ManagedResource Notified 2 observers of /schema/analysis/stopwords/english
[junit4] 2> 256030 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded LinkedHashMap at path _schema_analysis_synonyms_english.json using ZooKeeperStorageIO:path=/configs/testWithUploadDir
[junit4] 2> 256030 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.ManagedResource Loaded initArgs {ignoreCase=true,format=solr} for /schema/analysis/synonyms/english
[junit4] 2> 256032 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.s.a.ManagedSynonymFilterFactory Loaded 3 synonym mappings for /schema/analysis/synonyms/english
[junit4] 2> 256033 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.r.ManagedResource Notified 2 observers of /schema/analysis/synonyms/english
[junit4] 2> 256052 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/testWithUploadDir/managed-schema
[junit4] 2> 256068 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 256068 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 256092 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=default,field=text,classname=solr.DirectSolrSpellChecker,distanceMeasure=internal,accuracy=0.5,maxEdits=2,minPrefix=1,maxInspections=5,minQueryLength=4,maxQueryFrequency=0.01}
[junit4] 2> 256092 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.h.c.SuggestComponent Initializing SuggestComponent
[junit4] 2> 256258 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.s.s.SolrSuggester init: {name=mySuggester,lookupImpl=FuzzyLookupFactory,dictionaryImpl=DocumentDictionaryFactory,field=cat,weightField=price,suggestAnalyzerFieldType=string,buildOnStartup=false}
[junit4] 2> 256303 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.s.s.SolrSuggester Dictionary loaded with params: {name=mySuggester,lookupImpl=FuzzyLookupFactory,dictionaryImpl=DocumentDictionaryFactory,field=cat,weightField=price,suggestAnalyzerFieldType=string,buildOnStartup=false}
[junit4] 2> 256347 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 256347 INFO (searcherExecutor-848-thread-1-processing-n:192.168.57.18:8983_solr x:collection1 c:collection1) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.QuerySenderListener QuerySenderListener sending requests to Searcher@6d8c5bb7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 256348 INFO (coreLoadExecutor-847-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556279232115507200
[junit4] 2> 256353 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1889831623
[junit4] 2> 256353 INFO (searcherExecutor-848-thread-1-processing-n:192.168.57.18:8983_solr x:collection1 c:collection1) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=static+firstSearcher+warming+in+solrconfig.xml&distrib=false&event=firstSearcher} hits=0 status=0 QTime=3
[junit4] 2> 256353 INFO (searcherExecutor-848-thread-1-processing-n:192.168.57.18:8983_solr x:collection1 c:collection1) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.QuerySenderListener QuerySenderListener done.
[junit4] 2> 256353 INFO (searcherExecutor-848-thread-1-processing-n:192.168.57.18:8983_solr x:collection1 c:collection1) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 256353 INFO (searcherExecutor-848-thread-1-processing-n:192.168.57.18:8983_solr x:collection1 c:collection1) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 256355 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@4cd543ca
[junit4] 2> 256365 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: now flush at close
[junit4] 2> 256366 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: start flush: applyAllDeletes=true
[junit4] 2> 256366 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: index before flush
[junit4] 2> 256366 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [DW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: startFullFlush
[junit4] 2> 256367 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [DW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr finishFullFlush success=true
[junit4] 2> 256367 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: apply all deletes during flush
[junit4] 2> 256367 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: now apply all deletes for all segments maxDoc=0
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [BD][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: prune sis=segments_1: minGen=9223372036854775807 packetCount=0
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [MS][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: initDynamicDefaults spins=true maxThreadCount=1 maxMergeCount=6
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [MS][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: updateMergeThreads ioThrottle=true targetMBPerSec=10240.0 MB/sec
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [MS][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: now merge
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [MS][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: index:
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [MS][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: no more merges pending; now return
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: waitForMerges
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: waitForMerges done
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: commit: start
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: commit: enter lock
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: commit: now prepare
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: prepareCommit: flush
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: index before flush
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [DW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: startFullFlush
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: apply all deletes during flush
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: now apply all deletes for all segments maxDoc=0
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [BD][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: prune sis=segments_1: minGen=9223372036854775807 packetCount=0
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [DW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr finishFullFlush success=true
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: startCommit(): start
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: skip startCommit(): no changes pending
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: delete []
[junit4] 2> 256368 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: commit: pendingCommit == null; skip
[junit4] 2> 256369 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: commit: took 0.4 msec
[junit4] 2> 256369 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: commit: done
[junit4] 2> 256369 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: rollback
[junit4] 2> 256369 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: all running merges have aborted
[junit4] 2> 256369 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: rollback: done finish merges
[junit4] 2> 256369 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [DW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: abort
[junit4] 2> 256369 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [DW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: done abort success=true
[junit4] 2> 256369 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IW][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: rollback: infos=
[junit4] 2> 256369 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: now checkpoint "" [0 segments ; isCommit = false]
[junit4] 2> 256369 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: delete []
[junit4] 2> 256369 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: 0 msec to checkpoint
[junit4] 2> 256369 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.LoggingInfoStream [IFD][coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr]: delete []
[junit4] 2> 256370 INFO (searcherExecutor-848-thread-1-processing-n:192.168.57.18:8983_solr x:collection1 c:collection1) [n:192.168.57.18:8983_solr c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6d8c5bb7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 256377 INFO (coreCloseExecutor-853-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 256378 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.Overseer Overseer (id=97267451825029122-192.168.57.18:8983_solr-n_0000000000) closing
[junit4] 2> 256379 INFO (OverseerStateUpdate-97267451825029122-192.168.57.18:8983_solr-n_0000000000) [n:192.168.57.18:8983_solr ] o.a.s.c.Overseer Overseer Loop exiting : 192.168.57.18:8983_solr
[junit4] 2> 256381 WARN (zkCallback-234-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 257368 ERROR (coreZkRegister-840-thread-1-processing-n:192.168.57.18:8983_solr x:collection1 c:collection1) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController Error getting leader from zk
[junit4] 2> org.apache.solr.common.SolrException: CoreContainer is closed
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1037)
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1004)
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:960)
[junit4] 2> at org.apache.solr.cloud.ZkController.register(ZkController.java:899)
[junit4] 2> at org.apache.solr.cloud.ZkController.register(ZkController.java:843)
[junit4] 2> at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:181)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 257368 ERROR (coreZkRegister-840-thread-1-processing-n:192.168.57.18:8983_solr x:collection1 c:collection1) [n:192.168.57.18:8983_solr c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkContainer :org.apache.solr.common.SolrException: Error getting leader from zk for shard shard1
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:993)
[junit4] 2> at org.apache.solr.cloud.ZkController.register(ZkController.java:899)
[junit4] 2> at org.apache.solr.cloud.ZkController.register(ZkController.java:843)
[junit4] 2> at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:181)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: org.apache.solr.common.SolrException: CoreContainer is closed
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1037)
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1004)
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:960)
[junit4] 2> ... 7 more
[junit4] 2>
[junit4] 2> 257368 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 257370 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1889831623
[junit4] 2> 257371 ERROR (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.Overseer Could not create Overseer node
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$9.execute(SolrZkClient.java:391)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$9.execute(SolrZkClient.java:388)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:388)
[junit4] 2> at org.apache.solr.cloud.Overseer.createOverseerNode(Overseer.java:728)
[junit4] 2> at org.apache.solr.cloud.Overseer.getStateUpdateQueue(Overseer.java:601)
[junit4] 2> at org.apache.solr.cloud.Overseer.getStateUpdateQueue(Overseer.java:588)
[junit4] 2> at org.apache.solr.cloud.ZkController.publishNodeAsDown(ZkController.java:2407)
[junit4] 2> at org.apache.solr.core.CoreContainer.shutdown(CoreContainer.java:628)
[junit4] 2> at org.apache.solr.cloud.TestZkChroot.tearDown(TestZkChroot.java:57)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:965)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 257371 WARN (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.ZkController Could not publish node as down: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer
[junit4] 2> 257372 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 257373 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46584 46584
[junit4] 2> 257402 INFO (Thread-539) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46584 46584
[junit4] 2> 257403 WARN (Thread-539) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 3 /foo/bar3/overseer/queue
[junit4] 2> 3 /foo/bar3/overseer/collection-queue-work
[junit4] 2> 2 /foo/bar3/overseer/queue-work
[junit4] 2>
[junit4] 2> 257404 INFO (TEST-TestZkChroot.testWithUploadDir-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.SolrTestCaseJ4 ###Ending testWithUploadDir
[junit4] 2> 257406 INFO (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.SolrTestCaseJ4 ###Starting testNoBootstrapConf
[junit4] 2> 257407 INFO (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 257407 INFO (Thread-541) [n:192.168.57.18:8983_solr ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 257407 INFO (Thread-541) [n:192.168.57.18:8983_solr ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 257507 INFO (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.ZkTestServer start zk server on port:39947
[junit4] 2> 257691 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 259243 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 261180 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 262942 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 264905 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 266348 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 268201 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 269394 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 271290 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 272710 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 274600 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 276151 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 277585 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 279651 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 281534 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 283265 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 284945 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 286814 WARN (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]-SendThread(127.0.0.1:39947)) [n:192.168.57.18:8983_solr ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:39947, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:112)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
[junit4] 2> 288175 INFO (TEST-TestZkChroot.testNoBootstrapConf-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39947 39947
[junit4] 2> 288181 INFO (Thread-541) [n:192.168.57.18:8983_solr ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39947 39947
[junit4] 2> 288182 IN
[...truncated too long message...]
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: org.apache.solr.common.SolrException: CoreContainer is closed
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1037)
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1004)
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:960)
[junit4] 2> ... 7 more
[junit4] 2>
[junit4] 2> 295514 WARN (zkCallback-246-thread-1-processing-n:192.168.57.18:8983_solr) [n:192.168.57.18:8983_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 295515 INFO (TEST-TestZkChroot.testChrootBootstrap-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 295517 INFO (TEST-TestZkChroot.testChrootBootstrap-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2123947795
[junit4] 2> 295518 ERROR (TEST-TestZkChroot.testChrootBootstrap-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.Overseer Could not create Overseer node
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$9.execute(SolrZkClient.java:391)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$9.execute(SolrZkClient.java:388)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:388)
[junit4] 2> at org.apache.solr.cloud.Overseer.createOverseerNode(Overseer.java:728)
[junit4] 2> at org.apache.solr.cloud.Overseer.getStateUpdateQueue(Overseer.java:601)
[junit4] 2> at org.apache.solr.cloud.Overseer.getStateUpdateQueue(Overseer.java:588)
[junit4] 2> at org.apache.solr.cloud.ZkController.publishNodeAsDown(ZkController.java:2407)
[junit4] 2> at org.apache.solr.core.CoreContainer.shutdown(CoreContainer.java:628)
[junit4] 2> at org.apache.solr.cloud.TestZkChroot.tearDown(TestZkChroot.java:57)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:965)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 295518 WARN (TEST-TestZkChroot.testChrootBootstrap-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.ZkController Could not publish node as down: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer
[junit4] 2> 295518 INFO (TEST-TestZkChroot.testChrootBootstrap-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 295520 INFO (TEST-TestZkChroot.testChrootBootstrap-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43584 43584
[junit4] 2> 295549 INFO (Thread-544) [n:192.168.57.18:8983_solr ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43584 43584
[junit4] 2> 295550 WARN (Thread-544) [n:192.168.57.18:8983_solr ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 4 /foo/bar/overseer/queue
[junit4] 2> 2 /foo/bar/overseer/queue-work
[junit4] 2>
[junit4] 2> 295551 INFO (TEST-TestZkChroot.testChrootBootstrap-seed#[11E173D93FE82664]) [n:192.168.57.18:8983_solr ] o.a.s.SolrTestCaseJ4 ###Ending testChrootBootstrap
[junit4] 2> 295552 INFO (SUITE-TestZkChroot-seed#[11E173D93FE82664]-worker) [n:192.168.57.18:8983_solr ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestZkChroot_11E173D93FE82664-001
[junit4] 2> Jan 12, 2017 1:12:52 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=377, maxMBSortInHeap=6.192751851032053, sim=RandomSimilarity(queryNorm=false,coord=yes): {}, locale=ar-EG, timezone=Africa/Banjul
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=1,free=88436216,total=236068864
[junit4] 2> NOTE: All tests run in this JVM: [TestTestInjection, SortByFunctionTest, TestClusterProperties, TestDistributedGrouping, TestXmlQParserPlugin, TestUtils, SuggesterTSTTest, SolrJmxReporterTest, HdfsLockFactoryTest, FieldMutatingUpdateProcessorTest, DirectUpdateHandlerOptimizeTest, TestValueSourceCache, TestSolrCloudWithSecureImpersonation, MergeStrategyTest, ReplicationFactorTest, LoggingHandlerTest, HttpSolrCallGetCoreTest, TestSha256AuthenticationProvider, BufferStoreTest, RulesTest, CurrencyFieldOpenExchangeTest, DistributedQueryComponentOptimizationTest, TestBulkSchemaAPI, CollectionsAPISolrJTest, TestConfigSetsAPIZkFailure, TestFoldingMultitermQuery, TestZkChroot]
[junit4] Completed [32/680 (1!)] on J1 in 42.34s, 4 tests, 1 error <<< FAILURES!
[...truncated 64589 lines...]
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 612 -
Still unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/612/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
2 tests failed.
FAILED: org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testParallelCommitStream
Error Message:
expected:<5> but was:<0>
Stack Trace:
java.lang.AssertionError: expected:<5> but was:<0>
at __randomizedtesting.SeedInfo.seed([B10A63604D2FD55B:91E00160D16E3817]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testParallelCommitStream(StreamExpressionTest.java:3965)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
FAILED: org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testParallelUpdateStream
Error Message:
Error from server at http://127.0.0.1:40043/solr: collection already exists: parallelDestinationCollection
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:40043/solr: collection already exists: parallelDestinationCollection
at __randomizedtesting.SeedInfo.seed([B10A63604D2FD55B:D2E42082111B2E30]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:610)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:435)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:387)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1344)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1095)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1037)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
at org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testParallelUpdateStream(StreamExpressionTest.java:3466)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 13515 lines...]
[junit4] Suite: org.apache.solr.client.solrj.io.stream.StreamExpressionTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/init-core-data-001
[junit4] 2> 80192 INFO (SUITE-StreamExpressionTest-seed#[B10A63604D2FD55B]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 80192 INFO (SUITE-StreamExpressionTest-seed#[B10A63604D2FD55B]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001
[junit4] 2> 80192 INFO (SUITE-StreamExpressionTest-seed#[B10A63604D2FD55B]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 80193 INFO (Thread-174) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 80193 INFO (Thread-174) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 80293 INFO (SUITE-StreamExpressionTest-seed#[B10A63604D2FD55B]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:50119
[junit4] 2> 80305 INFO (jetty-launcher-102-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 80305 INFO (jetty-launcher-102-thread-3) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 80307 INFO (jetty-launcher-102-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 80307 INFO (jetty-launcher-102-thread-4) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 80310 INFO (jetty-launcher-102-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@385912da{/solr,null,AVAILABLE}
[junit4] 2> 80311 INFO (jetty-launcher-102-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@286a128d{HTTP/1.1,[http/1.1]}{127.0.0.1:48839}
[junit4] 2> 80311 INFO (jetty-launcher-102-thread-3) [ ] o.e.j.s.Server Started @83284ms
[junit4] 2> 80311 INFO (jetty-launcher-102-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=48839}
[junit4] 2> 80315 INFO (jetty-launcher-102-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2317ca16{/solr,null,AVAILABLE}
[junit4] 2> 80315 INFO (jetty-launcher-102-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@57568536{HTTP/1.1,[http/1.1]}{127.0.0.1:40043}
[junit4] 2> 80315 INFO (jetty-launcher-102-thread-2) [ ] o.e.j.s.Server Started @83288ms
[junit4] 2> 80315 INFO (jetty-launcher-102-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40043}
[junit4] 2> 80316 ERROR (jetty-launcher-102-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 80316 INFO (jetty-launcher-102-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.4.0
[junit4] 2> 80316 INFO (jetty-launcher-102-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 80316 INFO (jetty-launcher-102-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 80316 INFO (jetty-launcher-102-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-11T14:59:41.758Z
[junit4] 2> 80317 ERROR (jetty-launcher-102-thread-3) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 80317 INFO (jetty-launcher-102-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.4.0
[junit4] 2> 80317 INFO (jetty-launcher-102-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 80317 INFO (jetty-launcher-102-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 80317 INFO (jetty-launcher-102-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-11T14:59:41.759Z
[junit4] 2> 80325 INFO (jetty-launcher-102-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6bdf3f3d{/solr,null,AVAILABLE}
[junit4] 2> 80327 INFO (jetty-launcher-102-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3e131937{/solr,null,AVAILABLE}
[junit4] 2> 80328 INFO (jetty-launcher-102-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7aea6081{HTTP/1.1,[http/1.1]}{127.0.0.1:59699}
[junit4] 2> 80328 INFO (jetty-launcher-102-thread-4) [ ] o.e.j.s.Server Started @83301ms
[junit4] 2> 80328 INFO (jetty-launcher-102-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59699}
[junit4] 2> 80330 ERROR (jetty-launcher-102-thread-4) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 80330 INFO (jetty-launcher-102-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.4.0
[junit4] 2> 80330 INFO (jetty-launcher-102-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 80330 INFO (jetty-launcher-102-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 80331 INFO (jetty-launcher-102-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7a46626b{HTTP/1.1,[http/1.1]}{127.0.0.1:40531}
[junit4] 2> 80331 INFO (jetty-launcher-102-thread-1) [ ] o.e.j.s.Server Started @83304ms
[junit4] 2> 80331 INFO (jetty-launcher-102-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40531}
[junit4] 2> 80331 ERROR (jetty-launcher-102-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 80331 INFO (jetty-launcher-102-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 6.4.0
[junit4] 2> 80331 INFO (jetty-launcher-102-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 80331 INFO (jetty-launcher-102-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 80332 INFO (jetty-launcher-102-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-11T14:59:41.774Z
[junit4] 2> 80338 INFO (jetty-launcher-102-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-11T14:59:41.772Z
[junit4] 2> 80354 INFO (jetty-launcher-102-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 80357 INFO (jetty-launcher-102-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 80373 INFO (jetty-launcher-102-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 80376 INFO (jetty-launcher-102-thread-3) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 80377 INFO (jetty-launcher-102-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50119/solr
[junit4] 2> 80378 INFO (jetty-launcher-102-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 80381 INFO (jetty-launcher-102-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 80382 INFO (jetty-launcher-102-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50119/solr
[junit4] 2> 80387 INFO (jetty-launcher-102-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 80393 INFO (jetty-launcher-102-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50119/solr
[junit4] 2> 80396 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x1598e0aadc40004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 80398 INFO (jetty-launcher-102-thread-4) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 80398 INFO (jetty-launcher-102-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50119/solr
[junit4] 2> 80465 INFO (jetty-launcher-102-thread-2) [n:127.0.0.1:40043_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:40043_solr
[junit4] 2> 80469 INFO (jetty-launcher-102-thread-2) [n:127.0.0.1:40043_solr ] o.a.s.c.Overseer Overseer (id=97265043481296907-127.0.0.1:40043_solr-n_0000000000) starting
[junit4] 2> 80470 INFO (jetty-launcher-102-thread-4) [n:127.0.0.1:59699_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59699_solr
[junit4] 2> 80472 INFO (jetty-launcher-102-thread-3) [n:127.0.0.1:48839_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48839_solr
[junit4] 2> 80474 INFO (jetty-launcher-102-thread-1) [n:127.0.0.1:40531_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40531_solr
[junit4] 2> 80493 INFO (zkCallback-121-thread-2-processing-n:127.0.0.1:59699_solr) [n:127.0.0.1:59699_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 80494 INFO (zkCallback-120-thread-2-processing-n:127.0.0.1:40531_solr) [n:127.0.0.1:40531_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 80494 INFO (zkCallback-123-thread-2-processing-n:127.0.0.1:48839_solr) [n:127.0.0.1:48839_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 80495 INFO (zkCallback-120-thread-2-processing-n:127.0.0.1:40531_solr) [n:127.0.0.1:40531_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 80497 INFO (zkCallback-122-thread-1-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 80498 INFO (zkCallback-121-thread-1-processing-n:127.0.0.1:59699_solr) [n:127.0.0.1:59699_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 80504 INFO (jetty-launcher-102-thread-2) [n:127.0.0.1:40043_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40043_solr
[junit4] 2> 80508 INFO (zkCallback-123-thread-1-processing-n:127.0.0.1:48839_solr) [n:127.0.0.1:48839_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 80519 INFO (zkCallback-120-thread-1-processing-n:127.0.0.1:40531_solr) [n:127.0.0.1:40531_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 80520 INFO (zkCallback-121-thread-1-processing-n:127.0.0.1:59699_solr) [n:127.0.0.1:59699_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 80523 INFO (zkCallback-122-thread-1-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 80588 INFO (jetty-launcher-102-thread-1) [n:127.0.0.1:40531_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node1/.
[junit4] 2> 80652 INFO (jetty-launcher-102-thread-2) [n:127.0.0.1:40043_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node2/.
[junit4] 2> 80743 INFO (jetty-launcher-102-thread-4) [n:127.0.0.1:59699_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node4/.
[junit4] 2> 80752 INFO (jetty-launcher-102-thread-3) [n:127.0.0.1:48839_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node3/.
[junit4] 2> 80920 INFO (SUITE-StreamExpressionTest-seed#[B10A63604D2FD55B]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 80922 INFO (SUITE-StreamExpressionTest-seed#[B10A63604D2FD55B]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50119/solr ready
[junit4] 2> 80939 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=collection1_collection&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 80944 INFO (OverseerThreadFactory-313-thread-1-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.CreateCollectionCmd Create collection collection1_collection
[junit4] 2> 81090 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=collection1_collection_shard1_replica1&action=CREATE&numShards=2&collection=collection1_collection&shard=shard1&wt=javabin&version=2
[junit4] 2> 81090 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=collection1_collection_shard2_replica1&action=CREATE&numShards=2&collection=collection1_collection&shard=shard2&wt=javabin&version=2
[junit4] 2> 81206 INFO (zkCallback-122-thread-1-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 81234 INFO (zkCallback-123-thread-1-processing-n:127.0.0.1:48839_solr) [n:127.0.0.1:48839_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 82111 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 82115 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 82133 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.s.IndexSchema [collection1_collection_shard1_replica1] Schema name=test
[junit4] 2> 82133 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.s.IndexSchema [collection1_collection_shard2_replica1] Schema name=test
[junit4] 2> 82221 WARN (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.s.IndexSchema [collection1_collection_shard1_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 82223 WARN (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.s.IndexSchema [collection1_collection_shard2_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 82223 WARN (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
[junit4] 2> 82223 WARN (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiple copyFields (3)
[junit4] 2> 82224 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 82224 WARN (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
[junit4] 2> 82224 WARN (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiple copyFields (3)
[junit4] 2> 82225 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 82225 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_collection_shard1_replica1' using configuration from collection collection1_collection
[junit4] 2> 82226 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.c.SolrCore [[collection1_collection_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node2/collection1_collection_shard1_replica1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node2/./collection1_collection_shard1_replica1/data/]
[junit4] 2> 82231 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_collection_shard2_replica1' using configuration from collection collection1_collection
[junit4] 2> 82232 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.c.SolrCore [[collection1_collection_shard2_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node3/collection1_collection_shard2_replica1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node3/./collection1_collection_shard2_replica1/data/]
[junit4] 2> 82242 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 82242 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 82243 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 82243 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 82244 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@175a51be[collection1_collection_shard1_replica1] main]
[junit4] 2> 82246 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 82247 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 82247 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 82247 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556240697853476864
[junit4] 2> 82253 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 82253 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 82254 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 82254 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 82257 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 82257 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 82258 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:40043/solr/collection1_collection_shard1_replica1/
[junit4] 2> 82258 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 82258 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:40043/solr/collection1_collection_shard1_replica1/ has no replicas
[junit4] 2> 82259 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3b4cb282[collection1_collection_shard2_replica1] main]
[junit4] 2> 82260 INFO (searcherExecutor-322-thread-1-processing-n:127.0.0.1:40043_solr x:collection1_collection_shard1_replica1 s:shard1 c:collection1_collection) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.c.SolrCore [collection1_collection_shard1_replica1] Registered new searcher Searcher@175a51be[collection1_collection_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 82262 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 82263 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 82263 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 82264 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556240697871302656
[junit4] 2> 82264 INFO (searcherExecutor-323-thread-1-processing-n:127.0.0.1:48839_solr x:collection1_collection_shard2_replica1 s:shard2 c:collection1_collection) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.c.SolrCore [collection1_collection_shard2_replica1] Registered new searcher Searcher@3b4cb282[collection1_collection_shard2_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 82270 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:40043/solr/collection1_collection_shard1_replica1/ shard1
[junit4] 2> 82278 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 82278 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 82278 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:48839/solr/collection1_collection_shard2_replica1/
[junit4] 2> 82278 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 82278 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:48839/solr/collection1_collection_shard2_replica1/ has no replicas
[junit4] 2> 82287 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:48839/solr/collection1_collection_shard2_replica1/ shard2
[junit4] 2> 82395 INFO (zkCallback-123-thread-1-processing-n:127.0.0.1:48839_solr) [n:127.0.0.1:48839_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 82395 INFO (zkCallback-122-thread-2-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 82421 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 82425 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 x:collection1_collection_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=collection1_collection_shard1_replica1&action=CREATE&numShards=2&collection=collection1_collection&shard=shard1&wt=javabin&version=2} status=0 QTime=1336
[junit4] 2> 82447 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 82450 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 x:collection1_collection_shard2_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=collection1_collection_shard2_replica1&action=CREATE&numShards=2&collection=collection1_collection&shard=shard2&wt=javabin&version=2} status=0 QTime=1361
[junit4] 2> 82457 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 82556 INFO (zkCallback-123-thread-1-processing-n:127.0.0.1:48839_solr) [n:127.0.0.1:48839_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 82556 INFO (zkCallback-122-thread-3-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 82949 INFO (OverseerCollectionConfigSetProcessor-97265043481296907-127.0.0.1:40043_solr-n_0000000000) [n:127.0.0.1:40043_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 83458 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=collection1_collection&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=2520
[junit4] 2> 83459 INFO (SUITE-StreamExpressionTest-seed#[B10A63604D2FD55B]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1_collection failOnTimeout:true timeout (sec):30
[junit4] 2> 83460 INFO (SUITE-StreamExpressionTest-seed#[B10A63604D2FD55B]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1_collection
[junit4] 2> 83465 INFO (qtp1101808535-614) [n:127.0.0.1:40043_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :createalias with params collections=collection1_collection&name=collection1&action=CREATEALIAS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 83578 INFO (qtp1101808535-614) [n:127.0.0.1:40043_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collections=collection1_collection&name=collection1&action=CREATEALIAS&wt=javabin&version=2} status=0 QTime=113
[junit4] 2> 83619 INFO (TEST-StreamExpressionTest.testTopicStream-seed#[B10A63604D2FD55B]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testTopicStream
[junit4] 2> 83636 INFO (qtp1101808535-615) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 r:core_node1 x:collection1_collection_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_collection_shard1_replica1] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:48839/solr/collection1_collection_shard2_replica1/&wt=javabin&version=2}{deleteByQuery=*:* (-1556240699303657472)} 0 7
[junit4] 2> 83640 INFO (qtp1101808535-613) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 r:core_node1 x:collection1_collection_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 83640 INFO (qtp1101808535-613) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 r:core_node1 x:collection1_collection_shard1_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 83641 INFO (qtp1376078116-603) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 r:core_node2 x:collection1_collection_shard2_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 83641 INFO (qtp1376078116-603) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 r:core_node2 x:collection1_collection_shard2_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 83641 INFO (qtp1101808535-613) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 r:core_node1 x:collection1_collection_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 83642 INFO (qtp1101808535-613) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 r:core_node1 x:collection1_collection_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_collection_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:48839/solr/collection1_collection_shard2_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 83642 INFO (qtp1376078116-603) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 r:core_node2 x:collection1_collection_shard2_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 83642 INFO (qtp1376078116-603) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 r:core_node2 x:collection1_collection_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_collection_shard2_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:48839/solr/collection1_collection_shard2_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 83644 INFO (qtp1376078116-602) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 r:core_node2 x:collection1_collection_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_collection_shard2_replica1] webapp=/solr path=/update params={_stateVer_=collection1_collection:3&commit=true&wt=javabin&version=2}{deleteByQuery=*:* (-1556240699298414592),commit=} 0 21
[junit4] 2> 83648 INFO (TEST-StreamExpressionTest.testTopicStream-seed#[B10A63604D2FD55B]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testTopicStream
[junit4] IGNOR/A 0.08s J1 | StreamExpressionTest.testTopicStream
[junit4] > Assumption #1: got: <false>, expected: is <true>
[junit4] 2> 83691 INFO (TEST-StreamExpressionTest.testParallelExecutorStream-seed#[B10A63604D2FD55B]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testParallelExecutorStream
[junit4] 2> 83699 INFO (qtp1376078116-605) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 r:core_node2 x:collection1_collection_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_collection_shard2_replica1] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:40043/solr/collection1_collection_shard1_replica1/&wt=javabin&version=2}{deleteByQuery=*:* (-1556240699374960640)} 0 1
[junit4] 2> 83703 INFO (qtp1101808535-609) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 r:core_node1 x:collection1_collection_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 83704 INFO (qtp1101808535-609) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 r:core_node1 x:collection1_collection_shard1_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 83705 INFO (qtp1101808535-609) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 r:core_node1 x:collection1_collection_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 83705 INFO (qtp1101808535-609) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 r:core_node1 x:collection1_collection_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_collection_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:40043/solr/collection1_collection_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 83705 INFO (qtp1376078116-598) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 r:core_node2 x:collection1_collection_shard2_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 83705 INFO (qtp1376078116-598) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 r:core_node2 x:collection1_collection_shard2_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 83705 INFO (qtp1376078116-598) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 r:core_node2 x:collection1_collection_shard2_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 83705 INFO (qtp1376078116-598) [n:127.0.0.1:48839_solr c:collection1_collection s:shard2 r:core_node2 x:collection1_collection_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_collection_shard2_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:40043/solr/collection1_collection_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
[junit4] 2> 83706 INFO (qtp1101808535-625) [n:127.0.0.1:40043_solr c:collection1_collection s:shard1 r:core_node1 x:collection1_collection_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_collection_shard1_replica1] webapp=/solr path=/update params={_stateVer_=collection1_collection:3&commit=true&wt=javabin&version=2}{deleteByQuery=*:* (-1556240699371814912),commit=} 0 12
[junit4] 2> 83708 INFO (qtp1101808535-621) [n:127.0.0.1:40043_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=workQueue&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 83712 INFO (OverseerThreadFactory-313-thread-3-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.CreateCollectionCmd Create collection workQueue
[junit4] 2> 83713 INFO (OverseerCollectionConfigSetProcessor-97265043481296907-127.0.0.1:40043_solr-n_0000000000) [n:127.0.0.1:40043_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 83923 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=workQueue_shard2_replica1&action=CREATE&numShards=2&collection=workQueue&shard=shard2&wt=javabin&version=2
[junit4] 2> 83925 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=workQueue_shard1_replica1&action=CREATE&numShards=2&collection=workQueue&shard=shard1&wt=javabin&version=2
[junit4] 2> 84039 INFO (zkCallback-121-thread-1-processing-n:127.0.0.1:59699_solr) [n:127.0.0.1:59699_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/workQueue/state.json] for collection [workQueue] has occurred - updating... (live nodes size: [4])
[junit4] 2> 84039 INFO (zkCallback-123-thread-1-processing-n:127.0.0.1:48839_solr) [n:127.0.0.1:48839_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/workQueue/state.json] for collection [workQueue] has occurred - updating... (live nodes size: [4])
[junit4] 2> 84941 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 84942 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 84965 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.s.IndexSchema [workQueue_shard2_replica1] Schema name=test
[junit4] 2> 84965 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.s.IndexSchema [workQueue_shard1_replica1] Schema name=test
[junit4] 2> 85038 WARN (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.s.IndexSchema [workQueue_shard2_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 85039 WARN (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
[junit4] 2> 85040 WARN (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiple copyFields (3)
[junit4] 2> 85040 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 85045 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.c.CoreContainer Creating SolrCore 'workQueue_shard2_replica1' using configuration from collection workQueue
[junit4] 2> 85045 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.c.SolrCore [[workQueue_shard2_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node3/workQueue_shard2_replica1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node3/./workQueue_shard2_replica1/data/]
[junit4] 2> 85047 WARN (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.s.IndexSchema [workQueue_shard1_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 85055 WARN (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
[junit4] 2> 85055 WARN (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiple copyFields (3)
[junit4] 2> 85055 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 85070 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'workQueue_shard1_replica1' using configuration from collection workQueue
[junit4] 2> 85071 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.c.SolrCore [[workQueue_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node4/workQueue_shard1_replica1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node4/./workQueue_shard1_replica1/data/]
[junit4] 2> 85078 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 85078 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 85079 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 85079 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 85084 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@60ced0c2[workQueue_shard2_replica1] main]
[junit4] 2> 85084 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 85086 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 85087 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 85087 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 85087 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 85088 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 85088 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5292abab[workQueue_shard1_replica1] main]
[junit4] 2> 85088 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 85089 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556240700833529856
[junit4] 2> 85090 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 85091 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 85091 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 85091 INFO (searcherExecutor-334-thread-1-processing-n:127.0.0.1:59699_solr x:workQueue_shard1_replica1 s:shard1 c:workQueue) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.c.SolrCore [workQueue_shard1_replica1] Registered new searcher Searcher@5292abab[workQueue_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 85091 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556240700835627008
[junit4] 2> 85092 INFO (searcherExecutor-332-thread-1-processing-n:127.0.0.1:48839_solr x:workQueue_shard2_replica1 s:shard2 c:workQueue) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.c.SolrCore [workQueue_shard2_replica1] Registered new searcher Searcher@60ced0c2[workQueue_shard2_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 85102 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 85103 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 85103 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:59699/solr/workQueue_shard1_replica1/
[junit4] 2> 85103 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 85104 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:59699/solr/workQueue_shard1_replica1/ has no replicas
[junit4] 2> 85107 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 85107 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 85107 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:48839/solr/workQueue_shard2_replica1/
[junit4] 2> 85107 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 85107 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:48839/solr/workQueue_shard2_replica1/ has no replicas
[junit4] 2> 85115 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:59699/solr/workQueue_shard1_replica1/ shard1
[junit4] 2> 85119 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:48839/solr/workQueue_shard2_replica1/ shard2
[junit4] 2> 85225 INFO (zkCallback-121-thread-1-processing-n:127.0.0.1:59699_solr) [n:127.0.0.1:59699_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/workQueue/state.json] for collection [workQueue] has occurred - updating... (live nodes size: [4])
[junit4] 2> 85225 INFO (zkCallback-123-thread-1-processing-n:127.0.0.1:48839_solr) [n:127.0.0.1:48839_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/workQueue/state.json] for collection [workQueue] has occurred - updating... (live nodes size: [4])
[junit4] 2> 85267 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 85271 INFO (qtp653612463-631) [n:127.0.0.1:59699_solr c:workQueue s:shard1 x:workQueue_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=workQueue_shard1_replica1&action=CREATE&numShards=2&collection=workQueue&shard=shard1&wt=javabin&version=2} status=0 QTime=1346
[junit4] 2> 85273 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 85277 INFO (qtp1376078116-604) [n:127.0.0.1:48839_solr c:workQueue s:shard2 x:workQueue_shard2_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=workQueue_shard2_replica1&action=CREATE&numShards=2&collection=workQueue&shard=shard2&wt=javabin&version=2} status=0 QTime=1355
[junit4] 2> 85283 INFO (qtp1101808535-621) [n:127.0.0.1:40043_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 85381 INFO (zkCallback-123-thread-1-processing-n:127.0.0.1:48839_solr) [n:127.0.0.1:48839_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/workQueue/state.json] for collection [workQueue] has occurred - updating... (live nodes size: [4])
[junit4] 2> 85381 INFO (zkCallback-121-thread-1-processing-n:127.0.0.1:59699_solr) [n:127.0.0.1:59699_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/workQueue/state.json] for collection [workQueue] has occurred - updating... (live nodes size: [4])
[junit4] 2> 85719 INFO (OverseerCollectionConfigSetProcessor-97265043481296907-127.0.0.1:40043_solr-n_0000000000) [n:127.0.0.1:40043_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 86286 INFO (qtp1101808535-621) [n:127.0.0.1:40043_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=workQueue&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=2578
[junit4] 2> 86286 INFO (TEST-StreamExpressionTest.testParallelExecutorStream-seed#[B10A63604D2FD55B]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: workQueue failOnTimeout:true timeout (sec):30
[junit4] 2> 86287 INFO (TEST-StreamExpressionTest.testParallelExecutorStream-seed#[B10A63604D2FD55B]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: workQueue
[junit4] 2> 86290 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=mainCorpus&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 86294 INFO (OverseerThreadFactory-313-thread-4-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.CreateCollectionCmd Create collection mainCorpus
[junit4] 2> 86514 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=mainCorpus_shard2_replica1&action=CREATE&numShards=2&collection=mainCorpus&shard=shard2&wt=javabin&version=2
[junit4] 2> 86514 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=mainCorpus_shard1_replica1&action=CREATE&numShards=2&collection=mainCorpus&shard=shard1&wt=javabin&version=2
[junit4] 2> 86627 INFO (zkCallback-121-thread-1-processing-n:127.0.0.1:59699_solr) [n:127.0.0.1:59699_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/mainCorpus/state.json] for collection [mainCorpus] has occurred - updating... (live nodes size: [4])
[junit4] 2> 86628 INFO (zkCallback-122-thread-1-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/mainCorpus/state.json] for collection [mainCorpus] has occurred - updating... (live nodes size: [4])
[junit4] 2> 87541 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 87542 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 87563 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.s.IndexSchema [mainCorpus_shard1_replica1] Schema name=test
[junit4] 2> 87563 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.s.IndexSchema [mainCorpus_shard2_replica1] Schema name=test
[junit4] 2> 87644 WARN (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.s.IndexSchema [mainCorpus_shard2_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 87645 WARN (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.s.IndexSchema [mainCorpus_shard1_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 87645 WARN (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
[junit4] 2> 87645 WARN (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiple copyFields (3)
[junit4] 2> 87646 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 87646 WARN (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
[junit4] 2> 87646 WARN (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiple copyFields (3)
[junit4] 2> 87646 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 87647 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.c.CoreContainer Creating SolrCore 'mainCorpus_shard2_replica1' using configuration from collection mainCorpus
[junit4] 2> 87647 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'mainCorpus_shard1_replica1' using configuration from collection mainCorpus
[junit4] 2> 87648 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.c.SolrCore [[mainCorpus_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node2/mainCorpus_shard1_replica1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node2/./mainCorpus_shard1_replica1/data/]
[junit4] 2> 87648 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.c.SolrCore [[mainCorpus_shard2_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node4/mainCorpus_shard2_replica1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node4/./mainCorpus_shard2_replica1/data/]
[junit4] 2> 87662 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 87662 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 87663 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 87663 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 87665 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@198174b8[mainCorpus_shard1_replica1] main]
[junit4] 2> 87667 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 87668 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 87668 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 87668 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556240703537807360
[junit4] 2> 87670 INFO (searcherExecutor-343-thread-1-processing-n:127.0.0.1:40043_solr x:mainCorpus_shard1_replica1 s:shard1 c:mainCorpus) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.c.SolrCore [mainCorpus_shard1_replica1] Registered new searcher Searcher@198174b8[mainCorpus_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 87673 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 87673 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 87675 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 87675 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 87676 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6e3a9b95[mainCorpus_shard2_replica1] main]
[junit4] 2> 87678 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 87680 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 87680 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 87680 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 87680 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 87681 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:40043/solr/mainCorpus_shard1_replica1/
[junit4] 2> 87681 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 87681 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:40043/solr/mainCorpus_shard1_replica1/ has no replicas
[junit4] 2> 87680 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556240703550390272
[junit4] 2> 87680 INFO (searcherExecutor-342-thread-1-processing-n:127.0.0.1:59699_solr x:mainCorpus_shard2_replica1 s:shard2 c:mainCorpus) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.c.SolrCore [mainCorpus_shard2_replica1] Registered new searcher Searcher@6e3a9b95[mainCorpus_shard2_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 87693 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:40043/solr/mainCorpus_shard1_replica1/ shard1
[junit4] 2> 87697 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 87697 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 87697 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:59699/solr/mainCorpus_shard2_replica1/
[junit4] 2> 87697 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 87697 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:59699/solr/mainCorpus_shard2_replica1/ has no replicas
[junit4] 2> 87705 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:59699/solr/mainCorpus_shard2_replica1/ shard2
[junit4] 2> 87811 INFO (zkCallback-122-thread-2-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/mainCorpus/state.json] for collection [mainCorpus] has occurred - updating... (live nodes size: [4])
[junit4] 2> 87812 INFO (zkCallback-121-thread-1-processing-n:127.0.0.1:59699_solr) [n:127.0.0.1:59699_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/mainCorpus/state.json] for collection [mainCorpus] has occurred - updating... (live nodes size: [4])
[junit4] 2> 87850 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 87854 INFO (qtp1101808535-624) [n:127.0.0.1:40043_solr c:mainCorpus s:shard1 x:mainCorpus_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=mainCorpus_shard1_replica1&action=CREATE&numShards=2&collection=mainCorpus&shard=shard1&wt=javabin&version=2} status=0 QTime=1340
[junit4] 2> 87860 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 87863 INFO (qtp653612463-633) [n:127.0.0.1:59699_solr c:mainCorpus s:shard2 x:mainCorpus_shard2_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=mainCorpus_shard2_replica1&action=CREATE&numShards=2&collection=mainCorpus&shard=shard2&wt=javabin&version=2} status=0 QTime=1349
[junit4] 2> 87868 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 87981 INFO (zkCallback-121-thread-1-processing-n:127.0.0.1:59699_solr) [n:127.0.0.1:59699_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/mainCorpus/state.json] for collection [mainCorpus] has occurred - updating... (live nodes size: [4])
[junit4] 2> 87982 INFO (zkCallback-122-thread-1-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/mainCorpus/state.json] for collection [mainCorpus] has occurred - updating... (live nodes size: [4])
[junit4] 2> 88298 INFO (OverseerCollectionConfigSetProcessor-97265043481296907-127.0.0.1:40043_solr-n_0000000000) [n:127.0.0.1:40043_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 88869 INFO (qtp1101808535-616) [n:127.0.0.1:40043_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=mainCorpus&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=2580
[junit4] 2> 88870 INFO (TEST-StreamExpressionTest.testParallelExecutorStream-seed#[B10A63604D2FD55B]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: mainCorpus failOnTimeout:true timeout (sec):30
[junit4] 2> 88871 INFO (TEST-StreamExpressionTest.testParallelExecutorStream-seed#[B10A63604D2FD55B]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: mainCorpus
[junit4] 2> 88875 INFO (qtp1101808535-614) [n:127.0.0.1:40043_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=destination&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 88879 INFO (OverseerThreadFactory-313-thread-5-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.CreateCollectionCmd Create collection destination
[junit4] 2> 89098 INFO (qtp653612463-629) [n:127.0.0.1:59699_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=destination_shard2_replica1&action=CREATE&numShards=2&collection=destination&shard=shard2&wt=javabin&version=2
[junit4] 2> 89098 INFO (qtp1376078116-601) [n:127.0.0.1:48839_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=destination_shard1_replica1&action=CREATE&numShards=2&collection=destination&shard=shard1&wt=javabin&version=2
[junit4] 2> 89210 INFO (zkCallback-123-thread-1-processing-n:127.0.0.1:48839_solr) [n:127.0.0.1:48839_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/destination/state.json] for collection [destination] has occurred - updating... (live nodes size: [4])
[junit4] 2> 89210 INFO (zkCallback-121-thread-1-processing-n:127.0.0.1:59699_solr) [n:127.0.0.1:59699_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/destination/state.json] for collection [destination] has occurred - updating... (live nodes size: [4])
[junit4] 2> 90116 INFO (qtp653612463-629) [n:127.0.0.1:59699_solr c:destination s:shard2 x:destination_shard2_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 90117 INFO (qtp1376078116-601) [n:127.0.0.1:48839_solr c:destination s:shard1 x:destination_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 90146 INFO (qtp1376078116-601) [n:127.0.0.1:48839_solr c:destination s:shard1 x:destination_shard1_replica1] o.a.s.s.IndexSchema [destination_shard1_replica1] Schema name=test
[junit4] 2> 90146 INFO (qtp653612463-629) [n:127.0.0.1:59699_solr c:destination s:shard2 x:destination_shard2_replica1] o.a.s.s.IndexSchema [destination_shard2_replica1] Schema name=test
[junit4] 2> 90239 WARN (qtp1376078116-601) [n:127.0.0.1:48839_solr c:destination s:shard1 x:destination_shard1_replica1] o.a.s.s.IndexSchema [destination_shard1_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 90247 WARN (qtp1376078116-601) [n:127.0.0.1:48839_solr c:destination s:shard1 x:destination_shard1_replica1] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
[junit4] 2> 90247 WARN (qtp1376078116-601) [n:127.0.0.1:48839_solr c:destination s:shard1 x:destination_shard1_replica1] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiple copyFields (3)
[junit4] 2> 90248 INFO (qtp1376078116-601) [n:127.0.0.1:48839_solr c:destination s:shard1 x:destination_shard1_replica1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 90256 WARN (qtp653612463-629) [n:127.0.0.1:59699_solr c:destination s:shard2 x:destination_shard2_replica1] o.a.s.s.IndexSchema [destination_shard2_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 90257 WARN (qtp653612463-629) [n:127.0.0.1:59699_solr c:destination s:shard2 x:destination_shard2_replica1] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
[junit4] 2> 90258 WARN (qtp653612463-629) [n:127.0.0.1:59699_solr c:destination s:shard2 x:destination_shard2_replica1] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiple copyFields (3)
[junit4] 2> 90258 INFO (qtp653612463-629) [n:127.0.0.1:59699_solr c:destination s:shard2 x:destination_shard2_replica1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 90259 INFO (qtp1376078116-601) [n:127.0.0.1:48839_solr c:destination s:shard1 x:destination_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'destination_shard1_replica1' using configuration from collection destination
[junit4] 2> 90259 INFO (qtp1376078116-601) [n:127.0.0.1:48839_solr c:destination s:shard1 x:destination_shard1_replica1] o.a.s.c.SolrCore [[destination_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node3/destination_shard1_replica1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001/tempDir-001/node3/./destination_shard1_replica1/data/]
[junit4] 2> 90280 INFO (qtp653612463-629) [n:127.0.0.1:59699_solr c:destination s:
[...truncated too long message...]
4] 2> 164929 INFO (jetty-launcher-102-thread-7) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3e131937{/solr,null,UNAVAILABLE}
[junit4] 2> 164937 INFO (zkCallback-122-thread-13-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/parallelDestinationCollection/state.json] for collection [parallelDestinationCollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 164937 INFO (zkCallback-122-thread-11-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 164937 INFO (zkCallback-123-thread-1-processing-n:127.0.0.1:48839_solr) [n:127.0.0.1:48839_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 164942 INFO (zkCallback-122-thread-13-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 164949 INFO (zkCallback-123-thread-4-processing-n:127.0.0.1:48839_solr) [n:127.0.0.1:48839_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/parallelDestinationCollection/state.json] for collection [parallelDestinationCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 164949 INFO (zkCallback-123-thread-3-processing-n:127.0.0.1:48839_solr) [n:127.0.0.1:48839_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 164957 INFO (zkCallback-122-thread-13-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 164961 INFO (jetty-launcher-102-thread-6) [ ] o.a.s.c.Overseer Overseer (id=97265043481296907-127.0.0.1:40043_solr-n_0000000000) closing
[junit4] 2> 164961 WARN (OverseerStateUpdate-97265043481296907-127.0.0.1:40043_solr-n_0000000000) [n:127.0.0.1:40043_solr ] o.a.s.c.Overseer Bad version writing to ZK using compare-and-set, will force refresh cluster state: KeeperErrorCode = BadVersion for /collections/parallelDestinationCollection/state.json
[junit4] 2> 164961 INFO (OverseerStateUpdate-97265043481296907-127.0.0.1:40043_solr-n_0000000000) [n:127.0.0.1:40043_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:40043_solr
[junit4] 2> 164969 WARN (zkCallback-122-thread-11-processing-n:127.0.0.1:40043_solr) [n:127.0.0.1:40043_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 164971 INFO (jetty-launcher-102-thread-6) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 164972 INFO (jetty-launcher-102-thread-6) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2317ca16{/solr,null,UNAVAILABLE}
[junit4] 2> 166419 WARN (zkCallback-120-thread-1-processing-n:127.0.0.1:40531_solr) [n:127.0.0.1:40531_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 166420 INFO (jetty-launcher-102-thread-5) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 166420 INFO (jetty-launcher-102-thread-5) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6bdf3f3d{/solr,null,UNAVAILABLE}
[junit4] 2> 166449 WARN (zkCallback-123-thread-4-processing-n:127.0.0.1:48839_solr) [n:127.0.0.1:48839_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /collections/parallelDestinationCollection/state.json]
[junit4] 2> 166449 WARN (zkCallback-123-thread-3-processing-n:127.0.0.1:48839_solr) [n:127.0.0.1:48839_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /collections/collection1_collection/state.json]
[junit4] 2> 166455 WARN (zkCallback-123-thread-1-processing-n:127.0.0.1:48839_solr) [n:127.0.0.1:48839_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 166456 INFO (jetty-launcher-102-thread-8) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 166456 INFO (jetty-launcher-102-thread-8) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@385912da{/solr,null,UNAVAILABLE}
[junit4] 2> 166463 INFO (SUITE-StreamExpressionTest-seed#[B10A63604D2FD55B]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50119 50119
[junit4] 2> 166488 INFO (Thread-174) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50119 50119
[junit4] 2> 166490 WARN (Thread-174) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 145 /solr/aliases.json
[junit4] 2> 145 /solr/clusterprops.json
[junit4] 2> 11 /solr/configs/conf
[junit4] 2> 4 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 145 /solr/clusterstate.json
[junit4] 2> 4 /solr/collections/destination/state.json
[junit4] 2> 4 /solr/collections/destinationCollection/state.json
[junit4] 2> 4 /solr/collections/mainCorpus/state.json
[junit4] 2> 3 /solr/collections/workQueue/state.json
[junit4] 2> 2 /solr/collections/collection1_collection/state.json
[junit4] 2> 2 /solr/collections/parallelDestinationCollection/state.json
[junit4] 2> 2 /solr/overseer_elect/election/97265043481296907-127.0.0.1:40043_solr-n_0000000000
[junit4] 2> 2 /solr/collections/parallelDestinationCollection1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 145 /solr/live_nodes
[junit4] 2> 145 /solr/collections
[junit4] 2> 116 /solr/overseer/queue
[junit4] 2> 65 /solr/overseer/collection-queue-work
[junit4] 2> 21 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> 166490 INFO (SUITE-StreamExpressionTest-seed#[B10A63604D2FD55B]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_B10A63604D2FD55B-001
[junit4] 2> Jan 11, 2017 3:01:07 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {expr_s=PostingsFormat(name=Memory doPackFST= true), i_multi=PostingsFormat(name=Memory doPackFST= false), join2_s=PostingsFormat(name=LuceneVarGapDocFreqInterval), multiDefault=PostingsFormat(name=LuceneVarGapDocFreqInterval), subject=PostingsFormat(name=LuceneVarGapDocFreqInterval), intDefault=Lucene50(blocksize=128), a_f=Lucene50(blocksize=128), join1_i=PostingsFormat(name=Memory doPackFST= true), a_i=PostingsFormat(name=Memory doPackFST= false), join3_i=PostingsFormat(name=Memory doPackFST= false), field_i=PostingsFormat(name=LuceneVarGapDocFreqInterval), level1_s=PostingsFormat(name=Memory doPackFST= true), body_t=PostingsFormat(name=Memory doPackFST= false), ident_s=Lucene50(blocksize=128), s_multi=PostingsFormat(name=Memory doPackFST= true), level2_s=Lucene50(blocksize=128), a_s=PostingsFormat(name=Memory doPackFST= true), _version_=Lucene50(blocksize=128), side_s=PostingsFormat(name=Memory doPackFST= false), id=PostingsFormat(name=Memory doPackFST= false), text=PostingsFormat(name=Memory doPackFST= true), timestamp=PostingsFormat(name=LuceneVarGapDocFreqInterval)}, docValues:{expr_s=DocValuesFormat(name=Lucene54), i_multi=DocValuesFormat(name=Memory), join2_s=DocValuesFormat(name=Asserting), multiDefault=DocValuesFormat(name=Asserting), intDefault=DocValuesFormat(name=Lucene54), a_f=DocValuesFormat(name=Lucene54), join1_i=DocValuesFormat(name=Lucene54), a_i=DocValuesFormat(name=Memory), join3_i=DocValuesFormat(name=Memory), field_i=DocValuesFormat(name=Asserting), level1_s=DocValuesFormat(name=Lucene54), ident_s=DocValuesFormat(name=Lucene54), s_multi=DocValuesFormat(name=Lucene54), level2_s=DocValuesFormat(name=Lucene54), a_s=DocValuesFormat(name=Lucene54), side_s=DocValuesFormat(name=Memory), id=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1516, maxMBSortInHeap=5.771587723037907, sim=RandomSimilarity(queryNorm=false,coord=no): {}, locale=en-US, timezone=Indian/Mahe
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=1,free=67451688,total=166539264
[junit4] 2> NOTE: All tests run in this JVM: [TestSuggesterResponse, TestConfigSetAdminRequest, TestJsonRecordReader, ConcurrentUpdateSolrClientTest, GraphTest, TestLBHttpSolrClient, SolrExampleBinaryTest, TestZkConfigManager, GetByIdTest, QueryResponseTest, SolrExceptionTest, ContentStreamTest, TestClusteringResponse, MergeIndexesEmbeddedTest, LargeVolumeEmbeddedTest, CommonParamsTest, StreamExpressionTest]
[junit4] Completed [65/90 (1!)] on J1 in 86.57s, 52 tests, 1 failure, 1 error, 9 skipped <<< FAILURES!
[...truncated 61896 lines...]
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 611 -
Failure!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/611/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC
All tests passed
Build Log:
[...truncated 67939 lines...]
-ecj-javadoc-lint-tests:
[mkdir] Created dir: /var/tmp/ecj1036041760
[ecj-lint] Compiling 745 source files to /var/tmp/ecj1036041760
[ecj-lint] invalid Class-Path header in manifest of jar file: /export/home/jenkins/.ivy2/cache/org.restlet.jee/org.restlet/jars/org.restlet-2.3.0.jar
[ecj-lint] invalid Class-Path header in manifest of jar file: /export/home/jenkins/.ivy2/cache/org.restlet.jee/org.restlet.ext.servlet/jars/org.restlet.ext.servlet-2.3.0.jar
[ecj-lint] ----------
[ecj-lint] 1. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (at line 266)
[ecj-lint] ZkStateReader reader = new ZkStateReader(zkClient);
[ecj-lint] ^^^^^^
[ecj-lint] Resource leak: 'reader' is never closed
[ecj-lint] ----------
[ecj-lint] 2. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (at line 316)
[ecj-lint] ZkStateReader reader = new ZkStateReader(zkClient);
[ecj-lint] ^^^^^^
[ecj-lint] Resource leak: 'reader' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 3. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java (at line 146)
[ecj-lint] HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory();
[ecj-lint] ^^^^^^^^^^^
[ecj-lint] Resource leak: 'hdfsFactory' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 4. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/handler/admin/SecurityConfHandlerTest.java (at line 53)
[ecj-lint] BasicAuthPlugin basicAuth = new BasicAuthPlugin();
[ecj-lint] ^^^^^^^^^
[ecj-lint] Resource leak: 'basicAuth' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 5. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java (at line 163)
[ecj-lint] SolrClient client = random().nextBoolean() ? collection1 : collection2;
[ecj-lint] ^^^^^^
[ecj-lint] Resource leak: 'client' is never closed
[ecj-lint] ----------
[ecj-lint] 6. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java (at line 221)
[ecj-lint] throw new AssertionError(q.toString() + ": " + e.getMessage(), e);
[ecj-lint] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: 'client' is not closed at this location
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 7. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java (at line 201)
[ecj-lint] Analyzer a1 = new WhitespaceAnalyzer();
[ecj-lint] ^^
[ecj-lint] Resource leak: 'a1' is never closed
[ecj-lint] ----------
[ecj-lint] 8. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java (at line 204)
[ecj-lint] OffsetWindowTokenFilter tots = new OffsetWindowTokenFilter(tokenStream);
[ecj-lint] ^^^^
[ecj-lint] Resource leak: 'tots' is never closed
[ecj-lint] ----------
[ecj-lint] 9. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java (at line 208)
[ecj-lint] Analyzer a2 = new WhitespaceAnalyzer();
[ecj-lint] ^^
[ecj-lint] Resource leak: 'a2' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 10. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/request/TestBinaryResponseWriter.java (at line 65)
[ecj-lint] NamedList res = (NamedList) new JavaBinCodec().unmarshal(new ByteArrayInputStream(baos.toByteArray()));
[ecj-lint] ^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 11. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/response/TestBinaryResponseWriter.java (at line 63)
[ecj-lint] NamedList res = (NamedList) new JavaBinCodec().unmarshal(new ByteArrayInputStream(baos.toByteArray()));
[ecj-lint] ^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 12. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/response/TestJavabinTupleStreamParser.java (at line 72)
[ecj-lint] JavabinTupleStreamParser parser = new JavabinTupleStreamParser(new ByteArrayInputStream(bytes), true);
[ecj-lint] ^^^^^^
[ecj-lint] Resource leak: 'parser' is never closed
[ecj-lint] ----------
[ecj-lint] 13. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/response/TestJavabinTupleStreamParser.java (at line 173)
[ecj-lint] new JavaBinCodec().marshal(response.getValues(), baos);
[ecj-lint] ^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
[ecj-lint] ----------
[ecj-lint] 14. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/response/TestJavabinTupleStreamParser.java (at line 175)
[ecj-lint] Object o = new JavaBinCodec().unmarshal(new ByteArrayInputStream(bytes));
[ecj-lint] ^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
[ecj-lint] ----------
[ecj-lint] 15. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/response/TestJavabinTupleStreamParser.java (at line 178)
[ecj-lint] JavabinTupleStreamParser parser = new JavabinTupleStreamParser(new ByteArrayInputStream(bytes), false);
[ecj-lint] ^^^^^^
[ecj-lint] Resource leak: 'parser' is never closed
[ecj-lint] ----------
[ecj-lint] 16. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/response/TestJavabinTupleStreamParser.java (at line 192)
[ecj-lint] new JavaBinCodec().marshal(response.getValues(), baos);
[ecj-lint] ^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 17. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/response/TestPushWriter.java (at line 57)
[ecj-lint] m = (Map) new JavaBinCodec().unmarshal(new ByteArrayInputStream(baos.toByteArray()));
[ecj-lint] ^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 18. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/response/transform/TestSubQueryTransformer.java (at line 387)
[ecj-lint] unmarshalled = (NamedList<Object>) new JavaBinCodec().unmarshal(
[ecj-lint] ^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 19. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/search/TestDocSet.java (at line 241)
[ecj-lint] return loadfactor!=0 ? new HashDocSet(a,0,n,1/loadfactor) : new HashDocSet(a,0,n);
[ecj-lint] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
[ecj-lint] ----------
[ecj-lint] 20. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/search/TestDocSet.java (at line 531)
[ecj-lint] DocSet a = new BitDocSet(bs);
[ecj-lint] ^
[ecj-lint] Resource leak: 'a' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 21. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/security/TestPKIAuthenticationPlugin.java (at line 77)
[ecj-lint] final MockPKIAuthenticationPlugin mock = new MockPKIAuthenticationPlugin(null, nodeName);
[ecj-lint] ^^^^
[ecj-lint] Resource leak: 'mock' is never closed
[ecj-lint] ----------
[ecj-lint] 22. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/security/TestPKIAuthenticationPlugin.java (at line 130)
[ecj-lint] MockPKIAuthenticationPlugin mock1 = new MockPKIAuthenticationPlugin(null, nodeName) {
[ecj-lint] ^^^^^
[ecj-lint] Resource leak: 'mock1' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 23. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/security/TestRuleBasedAuthorizationPlugin.java (at line 365)
[ecj-lint] RuleBasedAuthorizationPlugin plugin = new RuleBasedAuthorizationPlugin();
[ecj-lint] ^^^^^^
[ecj-lint] Resource leak: 'plugin' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 24. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/security/TestSha256AuthenticationProvider.java (at line 49)
[ecj-lint] BasicAuthPlugin basicAuthPlugin = new BasicAuthPlugin();
[ecj-lint] ^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: 'basicAuthPlugin' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 25. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java (at line 42)
[ecj-lint] WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer();
[ecj-lint] ^^^^^^^^
[ecj-lint] Resource leak: 'analyzer' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 26. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java (at line 139)
[ecj-lint] IndexWriter w = new IndexWriter(d, newIndexWriterConfig(analyzer));
[ecj-lint] ^
[ecj-lint] Resource leak: 'w' is never closed
[ecj-lint] ----------
[ecj-lint] 27. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java (at line 172)
[ecj-lint] throw iae;
[ecj-lint] ^^^^^^^^^^
[ecj-lint] Resource leak: 'w' is not closed at this location
[ecj-lint] ----------
[ecj-lint] 28. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java (at line 178)
[ecj-lint] return;
[ecj-lint] ^^^^^^^
[ecj-lint] Resource leak: 'w' is not closed at this location
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 29. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java (at line 415)
[ecj-lint] new JavaBinCodec().marshal(topDocument, os);
[ecj-lint] ^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
[ecj-lint] ----------
[ecj-lint] 30. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java (at line 419)
[ecj-lint] SolrInputDocument result = (SolrInputDocument) new JavaBinCodec().unmarshal(is);
[ecj-lint] ^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 31. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java (at line 181)
[ecj-lint] EmbeddedSolrServer server1 = new EmbeddedSolrServer(h.getCoreContainer(), "split1");
[ecj-lint] ^^^^^^^
[ecj-lint] Resource leak: 'server1' is never closed
[ecj-lint] ----------
[ecj-lint] 32. WARNING in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java (at line 182)
[ecj-lint] EmbeddedSolrServer server2 = new EmbeddedSolrServer(h.getCoreContainer(), "split2");
[ecj-lint] ^^^^^^^
[ecj-lint] Resource leak: 'server2' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 33. ERROR in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactoryTest.java (at line 23)
[ecj-lint] import static org.mockito.Mockito.times;
[ecj-lint] ^^^^^^^^^^^^^^^^^^^^^^^^^
[ecj-lint] The import org.mockito.Mockito.times is never used
[ecj-lint] ----------
[ecj-lint] 33 problems (1 error, 32 warnings)
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:775: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:101: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build.xml:671: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:1998: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:2031: Compile failed; see the compiler error output for details.
Total time: 87 minutes 56 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 610 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/610/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130) at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102) at sun.reflect.GeneratedConstructorAccessor145.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:753) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:815) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1065) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:930) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:823) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:889) at org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:541) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
at sun.reflect.GeneratedConstructorAccessor145.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:753)
at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:815)
at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1065)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:930)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:823)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:889)
at org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:541)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([1D981EDE44192233]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:269)
at sun.reflect.GeneratedMethodAccessor25.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12435 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_1D981EDE44192233-001/init-core-data-001
[junit4] 2> 2505685 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 2505685 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_1D981EDE44192233-001/tempDir-001
[junit4] 2> 2505686 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2505686 INFO (Thread-4607) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2505686 INFO (Thread-4607) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2505786 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:34916
[junit4] 2> 2505794 INFO (jetty-launcher-3465-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2505794 INFO (jetty-launcher-3465-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2505797 INFO (jetty-launcher-3465-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@63a8a57e{/solr,null,AVAILABLE}
[junit4] 2> 2505798 INFO (jetty-launcher-3465-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@445688b{/solr,null,AVAILABLE}
[junit4] 2> 2505801 INFO (jetty-launcher-3465-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6d73c4d1{HTTP/1.1,[http/1.1]}{127.0.0.1:56377}
[junit4] 2> 2505801 INFO (jetty-launcher-3465-thread-2) [ ] o.e.j.s.Server Started @2509619ms
[junit4] 2> 2505801 INFO (jetty-launcher-3465-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=56377}
[junit4] 2> 2505802 INFO (jetty-launcher-3465-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@348805b6{HTTP/1.1,[http/1.1]}{127.0.0.1:39613}
[junit4] 2> 2505802 INFO (jetty-launcher-3465-thread-1) [ ] o.e.j.s.Server Started @2509620ms
[junit4] 2> 2505802 ERROR (jetty-launcher-3465-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2505802 INFO (jetty-launcher-3465-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39613}
[junit4] 2> 2505802 INFO (jetty-launcher-3465-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 2505802 ERROR (jetty-launcher-3465-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2505802 INFO (jetty-launcher-3465-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2505802 INFO (jetty-launcher-3465-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2505802 INFO (jetty-launcher-3465-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 2505803 INFO (jetty-launcher-3465-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2505803 INFO (jetty-launcher-3465-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-10T18:37:57.617Z
[junit4] 2> 2505803 INFO (jetty-launcher-3465-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2505803 INFO (jetty-launcher-3465-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-10T18:37:57.617Z
[junit4] 2> 2505806 INFO (jetty-launcher-3465-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2505806 INFO (jetty-launcher-3465-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2505814 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15989ac257e0002, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2505815 INFO (jetty-launcher-3465-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2505816 INFO (jetty-launcher-3465-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2505816 INFO (jetty-launcher-3465-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34916/solr
[junit4] 2> 2505817 INFO (jetty-launcher-3465-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34916/solr
[junit4] 2> 2505859 INFO (jetty-launcher-3465-thread-1) [n:127.0.0.1:39613_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:39613_solr
[junit4] 2> 2505861 INFO (jetty-launcher-3465-thread-2) [n:127.0.0.1:56377_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56377_solr
[junit4] 2> 2505861 INFO (jetty-launcher-3465-thread-1) [n:127.0.0.1:39613_solr ] o.a.s.c.Overseer Overseer (id=97260239421571077-127.0.0.1:39613_solr-n_0000000000) starting
[junit4] 2> 2505863 INFO (zkCallback-3476-thread-1-processing-n:127.0.0.1:39613_solr) [n:127.0.0.1:39613_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2505864 INFO (zkCallback-3475-thread-1-processing-n:127.0.0.1:56377_solr) [n:127.0.0.1:56377_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2505875 INFO (jetty-launcher-3465-thread-1) [n:127.0.0.1:39613_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39613_solr
[junit4] 2> 2505877 INFO (zkCallback-3476-thread-1-processing-n:127.0.0.1:39613_solr) [n:127.0.0.1:39613_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2505878 INFO (zkCallback-3475-thread-1-processing-n:127.0.0.1:56377_solr) [n:127.0.0.1:56377_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2505944 INFO (jetty-launcher-3465-thread-2) [n:127.0.0.1:56377_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_1D981EDE44192233-001/tempDir-001/node2/.
[junit4] 2> 2505946 INFO (jetty-launcher-3465-thread-1) [n:127.0.0.1:39613_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_1D981EDE44192233-001/tempDir-001/node1/.
[junit4] 2> 2506649 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2506651 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34916/solr ready
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 2506696 WARN (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 2506707 WARN (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2506709 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2506723 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_solaris.vm_47791_hdfs____2v9hvi/webapp
[junit4] 2> 2507274 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:47791
[junit4] 2> 2507412 WARN (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2507414 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2507428 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_43633_datanode____qqc6n2/webapp
[junit4] 2> 2507939 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43633
[junit4] 2> 2508025 WARN (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2508026 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2508041 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_51856_datanode____djxqn8/webapp
[junit4] 2> 2508116 INFO (IPC Server handler 5 on 50018) [ ] BlockStateChange BLOCK* processReport: from storage DS-2947d3ff-8654-4b6c-9ec6-0be22630d273 node DatanodeRegistration(127.0.0.1:46785, datanodeUuid=b29b7037-796d-4fa8-9f30-c52189c87322, infoPort=42866, infoSecurePort=0, ipcPort=50042, storageInfo=lv=-56;cid=testClusterID;nsid=1847105464;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2508116 INFO (IPC Server handler 5 on 50018) [ ] BlockStateChange BLOCK* processReport: from storage DS-5846e566-7941-4d71-ae18-814faa6d1264 node DatanodeRegistration(127.0.0.1:46785, datanodeUuid=b29b7037-796d-4fa8-9f30-c52189c87322, infoPort=42866, infoSecurePort=0, ipcPort=50042, storageInfo=lv=-56;cid=testClusterID;nsid=1847105464;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2508553 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:51856
[junit4] 2> 2508668 INFO (IPC Server handler 1 on 50018) [ ] BlockStateChange BLOCK* processReport: from storage DS-fc7aa681-d802-4e8b-b684-d97d2253f691 node DatanodeRegistration(127.0.0.1:40738, datanodeUuid=271aa0cc-1b25-4005-a8fb-68a0cce32d90, infoPort=52236, infoSecurePort=0, ipcPort=63015, storageInfo=lv=-56;cid=testClusterID;nsid=1847105464;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2508668 INFO (IPC Server handler 1 on 50018) [ ] BlockStateChange BLOCK* processReport: from storage DS-a40cc161-208c-498b-8dc7-9926ae53a712 node DatanodeRegistration(127.0.0.1:40738, datanodeUuid=271aa0cc-1b25-4005-a8fb-68a0cce32d90, infoPort=52236, infoSecurePort=0, ipcPort=63015, storageInfo=lv=-56;cid=testClusterID;nsid=1847105464;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2508821 INFO (TEST-HdfsRecoveryZkTest.test-seed#[1D981EDE44192233]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2508826 INFO (qtp746465824-20070) [n:127.0.0.1:56377_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2508830 INFO (OverseerThreadFactory-9314-thread-1-processing-n:127.0.0.1:39613_solr) [n:127.0.0.1:39613_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 2508941 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 2508951 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 2509058 INFO (zkCallback-3476-thread-1-processing-n:127.0.0.1:39613_solr) [n:127.0.0.1:39613_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2509058 INFO (zkCallback-3475-thread-1-processing-n:127.0.0.1:56377_solr) [n:127.0.0.1:56377_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2509953 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 2509980 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema [recoverytest_shard1_replica2] Schema name=minimal
[junit4] 2> 2509980 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 2509983 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2509984 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica2' using configuration from collection recoverytest
[junit4] 2> 2509989 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema [recoverytest_shard1_replica1] Schema name=minimal
[junit4] 2> 2509992 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2509993 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:50018/data
[junit4] 2> 2509993 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 2509993 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [[recoverytest_shard1_replica2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_1D981EDE44192233-001/tempDir-001/node2/recoverytest_shard1_replica2], dataDir=[hdfs://solaris-vm:50018/data/recoverytest/core_node1/data/]
[junit4] 2> 2509993 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica1' using configuration from collection recoverytest
[junit4] 2> 2509993 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:50018/data/recoverytest/core_node1/data/snapshot_metadata
[junit4] 2> 2509994 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:50018/data
[junit4] 2> 2509994 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 2509994 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [[recoverytest_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_1D981EDE44192233-001/tempDir-001/node1/recoverytest_shard1_replica1], dataDir=[hdfs://solaris-vm:50018/data/recoverytest/core_node2/data/]
[junit4] 2> 2509994 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:50018/data/recoverytest/core_node2/data/snapshot_metadata
[junit4] 2> 2510002 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2510002 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2510002 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2510002 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2510003 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2510003 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2510019 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2510022 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2510022 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:50018/data/recoverytest/core_node1/data
[junit4] 2> 2510023 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:50018/data/recoverytest/core_node2/data
[junit4] 2> 2510049 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:50018/data/recoverytest/core_node1/data/index
[junit4] 2> 2510049 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:50018/data/recoverytest/core_node2/data/index
[junit4] 2> 2510057 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2510057 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2510057 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2510063 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2510063 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2510063 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 2510112 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2510116 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2510163 INFO (IPC Server handler 1 on 50018) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:40738 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-5846e566-7941-4d71-ae18-814faa6d1264:NORMAL:127.0.0.1:46785|RBW], ReplicaUC[[DISK]DS-a40cc161-208c-498b-8dc7-9926ae53a712:NORMAL:127.0.0.1:40738|FINALIZED]]} size 0
[junit4] 2> 2510166 INFO (IPC Server handler 9 on 50018) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46785 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-5846e566-7941-4d71-ae18-814faa6d1264:NORMAL:127.0.0.1:46785|RBW], ReplicaUC[[DISK]DS-a40cc161-208c-498b-8dc7-9926ae53a712:NORMAL:127.0.0.1:40738|FINALIZED]]} size 0
[junit4] 2> 2510204 INFO (IPC Server handler 5 on 50018) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:40738 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-fc7aa681-d802-4e8b-b684-d97d2253f691:NORMAL:127.0.0.1:40738|RBW], ReplicaUC[[DISK]DS-2947d3ff-8654-4b6c-9ec6-0be22630d273:NORMAL:127.0.0.1:46785|RBW]]} size 0
[junit4] 2> 2510204 INFO (IPC Server handler 4 on 50018) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46785 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-fc7aa681-d802-4e8b-b684-d97d2253f691:NORMAL:127.0.0.1:40738|RBW], ReplicaUC[[DISK]DS-2947d3ff-8654-4b6c-9ec6-0be22630d273:NORMAL:127.0.0.1:46785|RBW]]} size 0
[junit4] 2> 2510210 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 2510210 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2510211 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 2510230 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 2510230 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2510230 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 2510235 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2510235 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2510243 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2510243 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2510259 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@8de204c[recoverytest_shard1_replica2] main]
[junit4] 2> 2510265 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4053cc87[recoverytest_shard1_replica1] main]
[junit4] 2> 2510265 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2510266 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2510266 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2510266 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556163835545518080
[junit4] 2> 2510267 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2510267 INFO (searcherExecutor-9319-thread-1-processing-n:127.0.0.1:56377_solr x:recoverytest_shard1_replica2 s:shard1 c:recoverytest) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [recoverytest_shard1_replica2] Registered new searcher Searcher@8de204c[recoverytest_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2510267 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2510268 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2510268 INFO (searcherExecutor-9320-thread-1-processing-n:127.0.0.1:39613_solr x:recoverytest_shard1_replica1 s:shard1 c:recoverytest) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [recoverytest_shard1_replica1] Registered new searcher Searcher@4053cc87[recoverytest_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2510268 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556163835547615232
[junit4] 2> 2510278 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2510278 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2510279 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/
[junit4] 2> 2510279 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:56377/solr START replicas=[http://127.0.0.1:39613/solr/recoverytest_shard1_replica1/] nUpdates=100
[junit4] 2> 2510280 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:56377/solr DONE. We have no versions. sync failed.
[junit4] 2> 2510284 INFO (qtp1281285570-20082) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.S.Request [recoverytest_shard1_replica1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 2510284 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 2510284 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 2510290 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/ shard1
[junit4] 2> 2510395 INFO (zkCallback-3475-thread-1-processing-n:127.0.0.1:56377_solr) [n:127.0.0.1:56377_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2510395 INFO (zkCallback-3476-thread-2-processing-n:127.0.0.1:39613_solr) [n:127.0.0.1:39613_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2510443 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2510446 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=1505
[junit4] 2> 2510552 INFO (zkCallback-3475-thread-1-processing-n:127.0.0.1:56377_solr) [n:127.0.0.1:56377_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2510552 INFO (zkCallback-3476-thread-1-processing-n:127.0.0.1:39613_solr) [n:127.0.0.1:39613_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2511284 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=2334
[junit4] 2> 2511288 INFO (qtp746465824-20070) [n:127.0.0.1:56377_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2511387 INFO (zkCallback-3476-thread-3-processing-n:127.0.0.1:39613_solr) [n:127.0.0.1:39613_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2511387 INFO (zkCallback-3475-thread-1-processing-n:127.0.0.1:56377_solr) [n:127.0.0.1:56377_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2512293 INFO (qtp746465824-20070) [n:127.0.0.1:56377_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=3468
[junit4] 2> 2512297 INFO (TEST-HdfsRecoveryZkTest.test-seed#[1D981EDE44192233]) [ ] o.a.s.c.RecoveryZkTest Indexing 1350 documents
[junit4] 2> 2512337 INFO (qtp1281285570-20080) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-0 (1556163837678321664)]} 0 20
[junit4] 2> 2512337 INFO (qtp746465824-20068) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 (1556163837678321664)]} 0 37
[junit4] 2> 2512341 INFO (qtp1281285570-20074) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-0 (1556163837679370240)]} 0 25
[junit4] 2> 2512342 INFO (qtp746465824-20071) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 (1556163837679370240)]} 0 41
[junit4] 2> 2512344 INFO (qtp1281285570-20082) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-1 (1556163837719216128)]} 0 3
[junit4] 2> 2512344 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 (1556163837719216128)]} 0 5
[junit4] 2> 2512352 INFO (qtp1281285570-20084) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-1 (1556163837725507584)]} 0 4
[junit4] 2> 2512352 INFO (qtp1281285570-20085) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-2 (1556163837726556160)]} 0 2
[junit4] 2> 2512352 INFO (qtp746465824-20076) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 (1556163837725507584)]} 0 6
[junit4] 2> 2512352 INFO (qtp746465824-20075) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 (1556163837726556160)]} 0 5
[junit4] 2> 2512359 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-2 (1556163837735993344)]} 0 3
[junit4] 2> 2512360 INFO (qtp1281285570-20083) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-1 (-1556163837735993345)]} 0 2
[junit4] 2> 2512360 INFO (qtp746465824-20070) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 (1556163837735993344)]} 0 5
[junit4] 2> 2512360 INFO (qtp746465824-20073) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-1 (-1556163837735993345)]} 0 5
[junit4] 2> 2512366 INFO (qtp1281285570-20080) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-1 (-1556163837743333376)]} 0 2
[junit4] 2> 2512366 INFO (qtp1281285570-20074) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-3 (1556163837743333377)]} 0 2
[junit4] 2> 2512366 INFO (qtp746465824-20068) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-1 (-1556163837743333376)]} 0 4
[junit4] 2> 2512367 INFO (qtp746465824-20071) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 (1556163837743333377)]} 0 5
[junit4] 2> 2512373 INFO (qtp1281285570-20082) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-4 (1556163837750673408)]} 0 2
[junit4] 2> 2512373 INFO (qtp1281285570-20084) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-3 (1556163837749624832)]} 0 2
[junit4] 2> 2512373 INFO (qtp746465824-20076) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 (1556163837750673408)]} 0 4
[junit4] 2> 2512373 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 (1556163837749624832)]} 0 5
[junit4] 2> 2512379 INFO (qtp1281285570-20085) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-4 (1556163837756964865)]} 0 2
[junit4] 2> 2512380 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-2 (-1556163837756964864)]} 0 2
[junit4] 2> 2512380 INFO (qtp746465824-20070) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 (1556163837756964865)]} 0 4
[junit4] 2> 2512380 INFO (qtp746465824-20075) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-2 (-1556163837756964864)]} 0 5
[junit4] 2> 2512386 INFO (qtp1281285570-20080) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-5 (1556163837764304897)]} 0 2
[junit4] 2> 2512386 INFO (qtp1281285570-20083) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-2 (-1556163837764304896)]} 0 2
[junit4] 2> 2512386 INFO (qtp746465824-20068) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 (1556163837764304897)]} 0 4
[junit4] 2> 2512386 INFO (qtp746465824-20073) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-2 (-1556163837764304896)]} 0 4
[junit4] 2> 2512392 INFO (qtp1281285570-20082) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-6 (1556163837770596352)]} 0 2
[junit4] 2> 2512392 INFO (qtp1281285570-20074) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-5 (1556163837770596353)]} 0 3
[junit4] 2> 2512393 INFO (qtp746465824-20071) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 (1556163837770596352)]} 0 4
[junit4] 2> 2512393 INFO (qtp746465824-20076) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 (1556163837770596353)]} 0 4
[junit4] 2> 2512399 INFO (qtp1281285570-20084) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-6 (-1556163837776887808)]} 0 3
[junit4] 2> 2512400 INFO (qtp1281285570-20085) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-6 (1556163837777936384)]} 0 3
[junit4] 2> 2512400 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-6 (-1556163837776887808)]} 0 5
[junit4] 2> 2512400 INFO (qtp746465824-20070) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-6 (1556163837777936384)]} 0 5
[junit4] 2> 2512405 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-7 (1556163837785276416)]} 0 2
[junit4] 2> 2512406 INFO (qtp1281285570-20080) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-6 (-1556163837785276417)]} 0 2
[junit4] 2> 2512406 INFO (qtp746465824-20075) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 (1556163837785276416)]} 0 4
[junit4] 2> 2512406 INFO (qtp746465824-20068) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-6 (-1556163837785276417)]} 0 4
[junit4] 2> 2512412 INFO (qtp1281285570-20083) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-8 (1556163837791567872)]} 0 2
[junit4] 2> 2512412 INFO (qtp1281285570-20082) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-7 (1556163837791567873)]} 0 2
[junit4] 2> 2512412 INFO (qtp746465824-20073) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 (1556163837791567872)]} 0 4
[junit4] 2> 2512413 INFO (qtp746465824-20071) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-7 (1556163837791567873)]} 0 4
[junit4] 2> 2512418 INFO (qtp1281285570-20084) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-8 (1556163837797859329)]} 0 2
[junit4] 2> 2512418 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-8 (1556163837797859329)]} 0 3
[junit4] 2> 2512419 INFO (qtp1281285570-20074) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-9 (1556163837797859328)]} 0 3
[junit4] 2> 2512419 INFO (qtp746465824-20076) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 (1556163837797859328)]} 0 5
[junit4] 2> 2512425 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-10 (1556163837805199360)]} 0 1
[junit4] 2> 2512425 INFO (qtp1281285570-20085) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-9 (1556163837804150784)]} 0 2
[junit4] 2> 2512425 INFO (qtp746465824-20075) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 (1556163837805199360)]} 0 3
[junit4] 2> 2512425 INFO (qtp746465824-20070) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-9 (1556163837804150784)]} 0 4
[junit4] 2> 2512431 INFO (qtp1281285570-20083) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-10 (1556163837811490816)]} 0 2
[junit4] 2> 2512431 INFO (qtp1281285570-20080) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-9 (-1556163837810442240)]} 0 2
[junit4] 2> 2512431 INFO (qtp746465824-20068) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-9 (-1556163837810442240)]} 0 4
[junit4] 2> 2512431 INFO (qtp746465824-20073) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-10 (1556163837811490816)]} 0 4
[junit4] 2> 2512438 INFO (qtp1281285570-20082) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-11 (1556163837817782273)]} 0 2
[junit4] 2> 2512438 INFO (qtp1281285570-20084) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-9 (-1556163837817782272)]} 0 2
[junit4] 2> 2512438 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-11 (1556163837817782273)]} 0 4
[junit4] 2> 2512438 INFO (qtp746465824-20071) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-9 (-1556163837817782272)]} 0 5
[junit4] 2> 2512444 INFO (qtp1281285570-20074) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-10 (-1556163837825122304)]} 0 2
[junit4] 2> 2512444 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-11 (1556163837825122305)]} 0 2
[junit4] 2> 2512445 INFO (qtp746465824-20076) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-10 (-1556163837825122304)]} 0 4
[junit4] 2> 2512445 INFO (qtp746465824-20075) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-11 (1556163837825122305)]} 0 4
[junit4] 2> 2512450 INFO (qtp1281285570-20085) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-10 (-1556163837831413761)]} 0 2
[junit4] 2> 2512450 INFO (qtp746465824-20068) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-10 (-1556163837831413761)]} 0 4
[junit4] 2> 2512451 INFO (qtp1281285570-20083) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-12 (1556163837831413760)]} 0 2
[junit4] 2> 2512451 INFO (qtp746465824-20070) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-12 (1556163837831413760)]} 0 4
[junit4] 2> 2512457 INFO (qtp1281285570-20080) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-12 (1556163837837705216)]} 0 3
[junit4] 2> 2512457 INFO (qtp1281285570-20082) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-12 (-1556163837838753792)]} 0 2
[junit4] 2> 2512458 INFO (qtp746465824-20073) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-12 (1556163837837705216)]} 0 5
[junit4] 2> 2512458 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-12 (-1556163837838753792)]} 0 4
[junit4] 2> 2512462 INFO (qtp1281285570-20084) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-13 (1556163837846093824)]} 0 1
[junit4] 2> 2512462 INFO (qtp1281285570-20074) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-12 (-1556163837846093825)]} 0 1
[junit4] 2> 2512463 INFO (qtp746465824-20071) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-13 (1556163837846093824)]} 0 3
[junit4] 2> 2512463 INFO (qtp746465824-20076) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-12 (-1556163837846093825)]} 0 3
[junit4] 2> 2512468 INFO (qtp1281285570-20085) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-13 (1556163837851336705)]} 0 1
[junit4] 2> 2512468 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-14 (1556163837851336704)]} 0 1
[junit4] 2> 2512468 INFO (qtp746465824-20068) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-13 (1556163837851336705)]} 0 3
[junit4] 2> 2512468 INFO (qtp746465824-20075) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-14 (1556163837851336704)]} 0 3
[junit4] 2> 2512473 INFO (qtp1281285570-20083) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-14 (1556163837856579585)]} 0 1
[junit4] 2> 2512473 INFO (qtp1281285570-20080) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-15 (1556163837856579584)]} 0 1
[junit4] 2> 2512473 INFO (qtp746465824-20073) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-14 (1556163837856579585)]} 0 3
[junit4] 2> 2512473 INFO (qtp746465824-20070) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-15 (1556163837856579584)]} 0 3
[junit4] 2> 2512481 INFO (qtp1281285570-20084) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-15 (1556163837861822464)]} 0 1
[junit4] 2> 2512481 INFO (qtp1281285570-20082) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-16 (1556163837861822465)]} 0 1
[junit4] 2> 2512482 INFO (qtp746465824-20071) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-16 (1556163837861822465)]} 0 6
[junit4] 2> 2512482 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-15 (1556163837861822464)]} 0 6
[junit4] 2> 2512487 INFO (qtp1281285570-20085) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-16 (1556163837871259649)]} 0 1
[junit4] 2> 2512487 INFO (qtp1281285570-20074) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-13 (-1556163837871259648)]} 0 1
[junit4] 2> 2512487 INFO (qtp746465824-20075) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-16 (1556163837871259649)]} 0 3
[junit4] 2> 2512487 INFO (qtp746465824-20076) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-13 (-1556163837871259648)]} 0 3
[junit4] 2> 2512492 INFO (qtp1281285570-20083) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-13 (-1556163837876502529)]} 0 1
[junit4] 2> 2512492 INFO (qtp746465824-20073) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-13 (-1556163837876502529)]} 0 3
[junit4] 2> 2512492 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-17 (1556163837876502528)]} 0 1
[junit4] 2> 2512493 INFO (qtp746465824-20068) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-17 (1556163837876502528)]} 0 3
[junit4] 2> 2512497 INFO (qtp1281285570-20080) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-17 (1556163837881745408)]} 0 1
[junit4] 2> 2512497 INFO (qtp746465824-20070) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-17 (1556163837881745408)]} 0 3
[junit4] 2> 2512498 INFO (qtp1281285570-20084) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-18 (1556163837881745409)]} 0 2
[junit4] 2> 2512498 INFO (qtp746465824-20071) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-18 (1556163837881745409)]} 0 3
[junit4] 2> 2512502 INFO (qtp1281285570-20082) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-18 (1556163837888036864)]} 0 1
[junit4] 2> 2512503 INFO (qtp1281285570-20085) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-19 (1556163837888036865)]} 0 1
[junit4] 2> 2512503 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-18 (1556163837888036864)]} 0 3
[junit4] 2> 2512503 INFO (qtp746465824-20075) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-19 (1556163837888036865)]} 0 3
[junit4] 2> 2512508 INFO (qtp1281285570-20074) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-15 (-1556163837893279744)]} 0 1
[junit4] 2> 2512508 INFO (qtp1281285570-20083) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-19 (1556163837893279745)]} 0 1
[junit4] 2> 2512508 INFO (qtp746465824-20076) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-15 (-1556163837893279744)]} 0 3
[junit4] 2> 2512508 INFO (qtp746465824-20073) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-19 (1556163837893279745)]} 0 3
[junit4] 2> 2512512 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-15 (-1556163837898522625)]} 0 1
[junit4] 2> 2512513 INFO (qtp746465824-20070) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-15 (-1556163837898522625)]} 0 2
[junit4] 2> 2512513 INFO (qtp1281285570-20080) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-20 (1556163837898522624)]} 0 1
[junit4] 2> 2512513 INFO (qtp746465824-20068) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-20 (1556163837898522624)]} 0 3
[junit4] 2> 2512518 INFO (qtp1281285570-20082) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-20 (1556163837902716928)]} 0 1
[junit4] 2> 2512518 INFO (qtp1281285570-20084) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-21 (1556163837903765504)]} 0 1
[junit4] 2> 2512518 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-21 (1556163837903765504)]} 0 3
[junit4] 2> 2512518 INFO (qtp746465824-20071) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-20 (1556163837902716928)]} 0 4
[junit4] 2> 2512525 INFO (qtp1281285570-20074) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-21 (1556163837910056961)]} 0 1
[junit4] 2> 2512525 INFO (qtp1281285570-20085) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-22 (1556163837910056960)]} 0 2
[junit4] 2> 2512525 INFO (qtp746465824-20075) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-22 (1556163837910056960)]} 0 4
[junit4] 2> 2512526 INFO (qtp746465824-20076) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-21 (1556163837910056961)]} 0 4
[junit4] 2> 2512530 INFO (qtp1281285570-20083) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-16 (-1556163837916348416)]} 0 0
[junit4] 2> 2512530 INFO (qtp746465824-20073) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-16 (-1556163837916348416)]} 0 2
[junit4] 2> 2512530 INFO (qtp1281285570-20079) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-22 (1556163837917396992)]} 0 1
[junit4] 2> 2512531 INFO (qtp746465824-20070) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-22 (1556163837917396992)]} 0 3
[junit4] 2> 2512535 INFO (qtp1281285570-20080) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-23 (1556163837921591296)]} 0 1
[junit4] 2> 2512536 INFO (qtp1281285570-20082) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-16 (-1556163837922639872)]} 0 0
[junit4] 2> 2512536 INFO (qtp746465824-20072) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-16 (-1556163837922639872)]} 0 3
[junit4] 2> 2512537 INFO (qtp746465824-20068) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-23 (1556163837921591296)]} 0 4
[junit4] 2> 2512541 INFO (qtp1281285570-20074) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-23 (1556163837927882752)]} 0 1
[junit4] 2> 2512541 INFO (qtp1281285570-20084) [n:127.0.0.1:39613_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:56377/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-17 (-1556163837927882753)]} 0 1
[junit4] 2> 2512541 INFO (qtp746465824-20075) [n:127.0.0.1:56377_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-17 (-1556163837927882753)]} 0 3
[junit4] 2> 2512541 INFO (qtp746465824-20071) [n:127.0.0.1:56377_solr c:recoverytest s:sh
[...truncated too long message...]
ilure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> 2529196 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34916 34916
[junit4] 2> 2529197 INFO (SUITE-HdfsRecoveryZkTest-seed#[1D981EDE44192233]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: test params are: codec=Lucene62, sim=RandomSimilarity(queryNorm=false,coord=crazy): {}, locale=de-AT, timezone=US/Samoa
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=2,free=242035896,total=524288000
[junit4] 2> NOTE: All tests run in this JVM: [MetricUtilsTest, BitVectorTest, TermsComponentTest, TestTolerantSearch, LeaderInitiatedRecoveryOnCommitTest, HttpPartitionTest, TestImplicitCoreProperties, BlockJoinFacetSimpleTest, TestSolrCoreSnapshots, FileUtilsTest, TestDFISimilarityFactory, TestReloadDeadlock, BlockDirectoryTest, DateRangeFieldTest, BasicAuthIntegrationTest, TestCharFilters, OutputWriterTest, TestSolrConfigHandler, AnalyticsMergeStrategyTest, DistributedMLTComponentTest, RuleEngineTest, TestDocumentBuilder, TestStressRecovery, ShardSplitTest, TestSubQueryTransformerCrossCore, PrimUtilsTest, DirectSolrConnectionTest, TestSegmentSorting, TestExtendedDismaxParser, ReplicaListTransformerTest, CollectionsAPIAsyncDistributedZkTest, DisMaxRequestHandlerTest, TestCSVLoader, DirectUpdateHandlerTest, JSONWriterTest, TestReplicationHandler, ResponseLogComponentTest, DateFieldTest, TestSort, TestBM25SimilarityFactory, TestStressVersions, ZkNodePropsTest, CachingDirectoryFactoryTest, TestFunctionQuery, AnalysisAfterCoreReloadTest, SharedFSAutoReplicaFailoverUtilsTest, TestDocBasedVersionConstraints, AlternateDirectoryTest, TestFastOutputStream, TestCursorMarkWithoutUniqueKey, TestAddFieldRealTimeGet, TestSolr4Spatial2, ZkStateWriterTest, TestNumericTerms64, DeleteLastCustomShardedReplicaTest, PeerSyncTest, AddBlockUpdateTest, TestTestInjection, TestCollapseQParserPlugin, TestJmxIntegration, TestSolrConfigHandlerCloud, DocumentAnalysisRequestHandlerTest, DirectSolrSpellCheckerTest, DocValuesTest, TestDocTermOrds, HighlighterMaxOffsetTest, TestFastLRUCache, OpenExchangeRatesOrgProviderTest, LeaderElectionTest, TestAuthorizationFramework, TestSQLHandler, TestElisionMultitermQuery, FullHLLTest, UniqFieldsUpdateProcessorFactoryTest, TestStressReorder, PluginInfoTest, DistributedQueryComponentCustomSortTest, DistributedQueueTest, TestDistribIDF, TestSolrIndexConfig, HardAutoCommitTest, HighlighterTest, RestartWhileUpdatingTest, TestLegacyFieldCache, ConnectionManagerTest, TestSchemaManager, TestPushWriter, ClassificationUpdateProcessorFactoryTest, HdfsBasicDistributedZkTest, VMParamsZkACLAndCredentialsProvidersTest, TestPerFieldSimilarityClassic, QueryElevationComponentTest, StandardRequestHandlerTest, TestOnReconnectListenerSupport, DistributedQueryElevationComponentTest, IndexBasedSpellCheckerTest, ActionThrottleTest, TestBlobHandler, CloudExitableDirectoryReaderTest, AssignTest, TestCoreContainer, HdfsNNFailoverTest, DocumentBuilderTest, TemplateUpdateProcessorTest, TestPseudoReturnFields, TestMissingGroups, TestReplicaProperties, TestClusterStateMutator, TestCSVResponseWriter, TestSchemaResource, TestBulkSchemaConcurrent, TestIBSimilarityFactory, DistributedFacetExistsSmallTest, RankQueryTest, TestRandomCollapseQParserPlugin, RequiredFieldsTest, TestRandomDVFaceting, SolrGraphiteReporterTest, TestHdfsBackupRestoreCore, TestHighFrequencyDictionaryFactory, TlogReplayBufferedWhileIndexingTest, DocValuesNotIndexedTest, SolrCoreTest, TestRuleBasedAuthorizationPlugin, PeerSyncWithIndexFingerprintCachingTest, TestManagedSynonymFilterFactory, TestFieldSortValues, TestSolrDeletionPolicy1, FastVectorHighlighterTest, TestSolrJ, MoreLikeThisHandlerTest, ZkControllerTest, XsltUpdateRequestHandlerTest, SampleTest, TestStressUserVersions, IndexSchemaRuntimeFieldTest, TestInitQParser, TestFaceting, TestSimpleQParserPlugin, TestDistributedSearch, BigEndianAscendingWordDeserializerTest, OverseerTaskQueueTest, TestConfigSetsAPIExclusivity, TestCryptoKeys, OutOfBoxZkACLAndCredentialsProvidersTest, TestGraphMLResponseWriter, SuggestComponentTest, TestRTimerTree, TestBinaryResponseWriter, SuggesterFSTTest, TestSchemaVersionResource, DistributedFacetPivotSmallTest, TestMinMaxOnMultiValuedField, SSLMigrationTest, PolyFieldTest, MigrateRouteKeyTest, TestPostingsSolrHighlighter, TestFieldCacheVsDocValues, TestHighlightDedupGrouping, PathHierarchyTokenizerFactoryTest, ReplaceNodeTest, TestXIncludeConfig, CustomCollectionTest, TestRawTransformer, CoreMergeIndexesAdminHandlerTest, UnloadDistributedZkTest, ShardRoutingTest, BasicZkTest, FullSolrCloudDistribCmdsTest, TestZkChroot, ZkCLITest, AutoCommitTest, TestDistributedGrouping, TermVectorComponentDistributedTest, DistributedTermsComponentTest, SpellCheckComponentTest, TestGroupingSearch, ConvertedLegacyTest, TestLazyCores, SoftAutoCommitTest, SuggesterTest, SpatialFilterTest, SolrCoreCheckLockOnStartupTest, FieldMutatingUpdateProcessorTest, QueryEqualityTest, StatelessScriptUpdateProcessorFactoryTest, TestSurroundQueryParser, TestQueryUtils, TestWriterPerf, PrimitiveFieldTypeTest, XmlUpdateRequestHandlerTest, TestIndexingPerformance, SolrPluginUtilsTest, TestCollationField, QueryParsingTest, JsonLoaderTest, TestPartialUpdateDeduplication, CSVRequestHandlerTest, TestComponentsName, TestLFUCache, SpellingQueryConverterTest, DOMUtilTest, ClusterStateTest, TestLRUCache, TestUtils, SliceStateTest, SystemInfoHandlerTest, CircularListTest, TestRTGBase, CursorPagingTest, CdcrReplicationDistributedZkTest, CdcrRequestHandlerTest, CdcrVersionReplicationTest, CleanupOldIndexTest, CollectionsAPISolrJTest, DistribDocExpirationUpdateProcessorTest, LeaderFailureAfterFreshStartTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, OverseerRolesTest, RecoveryAfterSoftCommitTest, SolrCLIZkUtilsTest, SolrCloudExampleTest, TestCloudRecovery, TestConfigSetsAPI, TestConfigSetsAPIZkFailure, TestDistribDocBasedVersion, TestLeaderElectionZkExpiry, TestLeaderInitiatedRecoveryThread, TestRandomRequestDistribution, TestRequestForwarding, TestRequestStatusCollectionAPI, TestSizeLimitedDistributedMap, TestTolerantUpdateProcessorCloud, HdfsRecoveryZkTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=HdfsRecoveryZkTest -Dtests.seed=1D981EDE44192233 -Dtests.slow=true -Dtests.locale=de-AT -Dtests.timezone=US/Samoa -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | HdfsRecoveryZkTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
[junit4] > at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
[junit4] > at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
[junit4] > at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
[junit4] > at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:753)
[junit4] > at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:815)
[junit4] > at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1065)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:930)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:823)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:889)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:541)
[junit4] > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([1D981EDE44192233]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:269)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] Completed [530/679 (1!)] on J0 in 54.63s, 1 test, 1 failure <<< FAILURES!
[...truncated 62964 lines...]
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 609 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/609/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130) at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:753) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:815) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1065) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:930) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:823) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:889) at org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:541) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:753)
at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:815)
at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1065)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:930)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:823)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:889)
at org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:541)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([8F6078DCBC564764]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:269)
at sun.reflect.GeneratedMethodAccessor32.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 10932 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_8F6078DCBC564764-001/init-core-data-001
[junit4] 2> 337654 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 337654 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_8F6078DCBC564764-001/tempDir-001
[junit4] 2> 337655 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 337655 INFO (Thread-1472) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 337656 INFO (Thread-1472) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 337756 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:48378
[junit4] 2> 337775 INFO (jetty-launcher-358-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 337776 INFO (jetty-launcher-358-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@73c6374c{/solr,null,AVAILABLE}
[junit4] 2> 337777 INFO (jetty-launcher-358-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@bb12b00{HTTP/1.1,[http/1.1]}{127.0.0.1:35451}
[junit4] 2> 337777 INFO (jetty-launcher-358-thread-1) [ ] o.e.j.s.Server Started @340691ms
[junit4] 2> 337777 INFO (jetty-launcher-358-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35451}
[junit4] 2> 337777 ERROR (jetty-launcher-358-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 337779 INFO (jetty-launcher-358-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 337779 INFO (jetty-launcher-358-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 337779 INFO (jetty-launcher-358-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 337779 INFO (jetty-launcher-358-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-10T07:52:37.133Z
[junit4] 2> 337782 INFO (jetty-launcher-358-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 337784 INFO (jetty-launcher-358-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7d72610b{/solr,null,AVAILABLE}
[junit4] 2> 337785 INFO (jetty-launcher-358-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4b254359{HTTP/1.1,[http/1.1]}{127.0.0.1:35132}
[junit4] 2> 337785 INFO (jetty-launcher-358-thread-2) [ ] o.e.j.s.Server Started @340698ms
[junit4] 2> 337785 INFO (jetty-launcher-358-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35132}
[junit4] 2> 337786 ERROR (jetty-launcher-358-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 337786 INFO (jetty-launcher-358-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 337786 INFO (jetty-launcher-358-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 337786 INFO (jetty-launcher-358-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 337786 INFO (jetty-launcher-358-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-10T07:52:37.140Z
[junit4] 2> 337790 INFO (jetty-launcher-358-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 337793 INFO (jetty-launcher-358-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 337800 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x159875d519a0002, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 337801 INFO (jetty-launcher-358-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 337802 INFO (jetty-launcher-358-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 337803 INFO (jetty-launcher-358-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48378/solr
[junit4] 2> 337803 INFO (jetty-launcher-358-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48378/solr
[junit4] 2> 337864 INFO (jetty-launcher-358-thread-1) [n:127.0.0.1:35451_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35451_solr
[junit4] 2> 337866 INFO (jetty-launcher-358-thread-1) [n:127.0.0.1:35451_solr ] o.a.s.c.Overseer Overseer (id=97257701835931654-127.0.0.1:35451_solr-n_0000000000) starting
[junit4] 2> 337884 INFO (jetty-launcher-358-thread-1) [n:127.0.0.1:35451_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35451_solr
[junit4] 2> 337884 INFO (jetty-launcher-358-thread-2) [n:127.0.0.1:35132_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35132_solr
[junit4] 2> 337897 INFO (zkCallback-369-thread-1-processing-n:127.0.0.1:35451_solr) [n:127.0.0.1:35451_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 337898 INFO (zkCallback-368-thread-1-processing-n:127.0.0.1:35132_solr) [n:127.0.0.1:35132_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 337986 INFO (jetty-launcher-358-thread-1) [n:127.0.0.1:35451_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_8F6078DCBC564764-001/tempDir-001/node1/.
[junit4] 2> 338056 INFO (jetty-launcher-358-thread-2) [n:127.0.0.1:35132_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_8F6078DCBC564764-001/tempDir-001/node2/.
[junit4] 2> 338187 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 338188 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48378/solr ready
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 338227 WARN (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 338238 WARN (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 338241 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 338255 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_solaris.vm_55967_hdfs____.m76el0/webapp
[junit4] 2> 338770 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:55967
[junit4] 2> 338948 WARN (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 338950 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 338968 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_55243_datanode____.4tytjc/webapp
[junit4] 2> 339530 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:55243
[junit4] 2> 339668 WARN (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 339670 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 339757 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_56081_datanode____.fujmsx/webapp
[junit4] 2> 339837 INFO (IPC Server handler 4 on 53824) [ ] BlockStateChange BLOCK* processReport: from storage DS-388c29c0-02d8-42ff-b504-f3be51d6e44f node DatanodeRegistration(127.0.0.1:60428, datanodeUuid=98aa4de7-31a0-43a8-90a6-2b685d402c56, infoPort=55758, infoSecurePort=0, ipcPort=59893, storageInfo=lv=-56;cid=testClusterID;nsid=2015539724;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
[junit4] 2> 339838 INFO (IPC Server handler 4 on 53824) [ ] BlockStateChange BLOCK* processReport: from storage DS-6a71e111-0022-4058-9f11-b137b4500501 node DatanodeRegistration(127.0.0.1:60428, datanodeUuid=98aa4de7-31a0-43a8-90a6-2b685d402c56, infoPort=55758, infoSecurePort=0, ipcPort=59893, storageInfo=lv=-56;cid=testClusterID;nsid=2015539724;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 340326 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:56081
[junit4] 2> 340517 INFO (IPC Server handler 6 on 53824) [ ] BlockStateChange BLOCK* processReport: from storage DS-84ce1c8f-a03f-42b6-adc3-33c4d89068c9 node DatanodeRegistration(127.0.0.1:36319, datanodeUuid=bef9753e-08d9-46de-8200-84be33112f12, infoPort=43185, infoSecurePort=0, ipcPort=47869, storageInfo=lv=-56;cid=testClusterID;nsid=2015539724;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
[junit4] 2> 340517 INFO (IPC Server handler 6 on 53824) [ ] BlockStateChange BLOCK* processReport: from storage DS-60d216df-913e-4dec-867b-fd33f184ee9c node DatanodeRegistration(127.0.0.1:36319, datanodeUuid=bef9753e-08d9-46de-8200-84be33112f12, infoPort=43185, infoSecurePort=0, ipcPort=47869, storageInfo=lv=-56;cid=testClusterID;nsid=2015539724;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 340674 INFO (TEST-HdfsRecoveryZkTest.test-seed#[8F6078DCBC564764]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 340685 INFO (qtp1775485046-3270) [n:127.0.0.1:35451_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 340692 INFO (OverseerThreadFactory-1191-thread-1-processing-n:127.0.0.1:35451_solr) [n:127.0.0.1:35451_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 340805 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 340819 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 340930 INFO (zkCallback-368-thread-1-processing-n:127.0.0.1:35132_solr) [n:127.0.0.1:35132_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 340930 INFO (zkCallback-369-thread-1-processing-n:127.0.0.1:35451_solr) [n:127.0.0.1:35451_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 341843 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 341852 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 341853 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema [recoverytest_shard1_replica1] Schema name=minimal
[junit4] 2> 341861 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 341864 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema [recoverytest_shard1_replica2] Schema name=minimal
[junit4] 2> 341865 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica1' using configuration from collection recoverytest
[junit4] 2> 341868 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 341879 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:53824/data
[junit4] 2> 341879 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 341880 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [[recoverytest_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_8F6078DCBC564764-001/tempDir-001/node1/recoverytest_shard1_replica1], dataDir=[hdfs://solaris-vm:53824/data/recoverytest/core_node1/data/]
[junit4] 2> 341880 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:53824/data/recoverytest/core_node1/data/snapshot_metadata
[junit4] 2> 341895 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 341895 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 341895 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 341920 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica2' using configuration from collection recoverytest
[junit4] 2> 341927 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:53824/data
[junit4] 2> 341927 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 341927 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [[recoverytest_shard1_replica2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_8F6078DCBC564764-001/tempDir-001/node2/recoverytest_shard1_replica2], dataDir=[hdfs://solaris-vm:53824/data/recoverytest/core_node2/data/]
[junit4] 2> 341928 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:53824/data/recoverytest/core_node2/data/snapshot_metadata
[junit4] 2> 341944 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 341946 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 341946 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 341946 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 341947 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:53824/data/recoverytest/core_node1/data
[junit4] 2> 342009 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:53824/data/recoverytest/core_node1/data/index
[junit4] 2> 342080 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 342080 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 342080 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 342207 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 342209 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:53824/data/recoverytest/core_node2/data
[junit4] 2> 342251 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:53824/data/recoverytest/core_node2/data/index
[junit4] 2> 342259 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 342259 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 342259 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 342333 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 342337 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 342531 INFO (IPC Server handler 7 on 53824) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:60428 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-60d216df-913e-4dec-867b-fd33f184ee9c:NORMAL:127.0.0.1:36319|RBW], ReplicaUC[[DISK]DS-6a71e111-0022-4058-9f11-b137b4500501:NORMAL:127.0.0.1:60428|FINALIZED]]} size 0
[junit4] 2> 342532 INFO (IPC Server handler 8 on 53824) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:36319 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-388c29c0-02d8-42ff-b504-f3be51d6e44f:NORMAL:127.0.0.1:60428|RBW], ReplicaUC[[DISK]DS-60d216df-913e-4dec-867b-fd33f184ee9c:NORMAL:127.0.0.1:36319|RBW]]} size 0
[junit4] 2> 342533 INFO (IPC Server handler 2 on 53824) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:60428 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-388c29c0-02d8-42ff-b504-f3be51d6e44f:NORMAL:127.0.0.1:60428|RBW], ReplicaUC[[DISK]DS-60d216df-913e-4dec-867b-fd33f184ee9c:NORMAL:127.0.0.1:36319|RBW]]} size 0
[junit4] 2> 342533 INFO (IPC Server handler 2 on 53824) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:36319 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-6a71e111-0022-4058-9f11-b137b4500501:NORMAL:127.0.0.1:60428|FINALIZED], ReplicaUC[[DISK]DS-84ce1c8f-a03f-42b6-adc3-33c4d89068c9:NORMAL:127.0.0.1:36319|RBW]]} size 0
[junit4] 2> 342566 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 342566 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 342567 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 342585 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 342586 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 342586 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 342597 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 342597 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 342602 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 342602 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 342615 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3eb03a8[recoverytest_shard1_replica1] main]
[junit4] 2> 342617 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 342617 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 342618 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 342618 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556123234569551872
[junit4] 2> 342624 INFO (searcherExecutor-1196-thread-1-processing-n:127.0.0.1:35451_solr x:recoverytest_shard1_replica1 s:shard1 c:recoverytest) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [recoverytest_shard1_replica1] Registered new searcher Searcher@3eb03a8[recoverytest_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 342628 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@5a2b586[recoverytest_shard1_replica2] main]
[junit4] 2> 342630 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 342631 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 342632 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 342632 INFO (searcherExecutor-1197-thread-1-processing-n:127.0.0.1:35132_solr x:recoverytest_shard1_replica2 s:shard1 c:recoverytest) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [recoverytest_shard1_replica2] Registered new searcher Searcher@5a2b586[recoverytest_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 342633 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556123234585280512
[junit4] 2> 342640 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9998ms
[junit4] 2> 342642 INFO (zkCallback-369-thread-3-processing-n:127.0.0.1:35451_solr) [n:127.0.0.1:35451_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 342645 INFO (zkCallback-368-thread-1-processing-n:127.0.0.1:35132_solr) [n:127.0.0.1:35132_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 343142 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 343142 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 343142 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/
[junit4] 2> 343142 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica1 url=http://127.0.0.1:35451/solr START replicas=[http://127.0.0.1:35132/solr/recoverytest_shard1_replica2/] nUpdates=100
[junit4] 2> 343142 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica1 url=http://127.0.0.1:35451/solr DONE. We have no versions. sync failed.
[junit4] 2> 343147 INFO (qtp2130827995-3279) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.c.S.Request [recoverytest_shard1_replica2] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 343148 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 343148 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 343153 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/ shard1
[junit4] 2> 343258 INFO (zkCallback-369-thread-1-processing-n:127.0.0.1:35451_solr) [n:127.0.0.1:35451_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 343258 INFO (zkCallback-368-thread-1-processing-n:127.0.0.1:35132_solr) [n:127.0.0.1:35132_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 343305 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 343308 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=2504
[junit4] 2> 343415 INFO (zkCallback-369-thread-3-processing-n:127.0.0.1:35451_solr) [n:127.0.0.1:35451_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 343416 INFO (zkCallback-368-thread-1-processing-n:127.0.0.1:35132_solr) [n:127.0.0.1:35132_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 343658 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=2840
[junit4] 2> 343668 INFO (qtp1775485046-3270) [n:127.0.0.1:35451_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 343763 INFO (zkCallback-369-thread-2-processing-n:127.0.0.1:35451_solr) [n:127.0.0.1:35451_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 343765 INFO (zkCallback-368-thread-1-processing-n:127.0.0.1:35132_solr) [n:127.0.0.1:35132_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 344669 INFO (qtp1775485046-3270) [n:127.0.0.1:35451_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=3986
[junit4] 2> 344680 INFO (TEST-HdfsRecoveryZkTest.test-seed#[8F6078DCBC564764]) [ ] o.a.s.c.RecoveryZkTest Indexing 1200 documents
[junit4] 2> 344702 INFO (OverseerCollectionConfigSetProcessor-97257701835931654-127.0.0.1:35451_solr-n_0000000000) [n:127.0.0.1:35451_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 344739 INFO (qtp2130827995-3278) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-0 (1556123236738007041)]} 0 39
[junit4] 2> 344740 INFO (qtp1775485046-3271) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 (1556123236738007041)]} 0 54
[junit4] 2> 344740 INFO (qtp2130827995-3280) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-0 (1556123236738007040)]} 0 40
[junit4] 2> 344741 INFO (qtp1775485046-3266) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 (1556123236738007040)]} 0 54
[junit4] 2> 344750 INFO (qtp2130827995-3285) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-0 (-1556123236798824448)]} 0 3
[junit4] 2> 344750 INFO (qtp2130827995-3283) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-0 (-1556123236797775872)]} 0 3
[junit4] 2> 344750 INFO (qtp1775485046-3267) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-0 (-1556123236797775872)]} 0 7
[junit4] 2> 344750 INFO (qtp1775485046-3268) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-0 (-1556123236798824448)]} 0 7
[junit4] 2> 344766 INFO (qtp2130827995-3279) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-1 (1556123236808261633)]} 0 9
[junit4] 2> 344767 INFO (qtp2130827995-3284) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-1 (1556123236808261632)]} 0 10
[junit4] 2> 344767 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 (1556123236808261632)]} 0 14
[junit4] 2> 344768 INFO (qtp1775485046-3270) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 (1556123236808261633)]} 0 15
[junit4] 2> 344779 INFO (qtp2130827995-3278) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-2 (1556123236828184576)]} 0 3
[junit4] 2> 344780 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-2 (1556123236827136000)]} 0 6
[junit4] 2> 344780 INFO (qtp1775485046-3271) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 (1556123236828184576)]} 0 7
[junit4] 2> 344780 INFO (qtp1775485046-3273) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 (1556123236827136000)]} 0 9
[junit4] 2> 344795 INFO (qtp2130827995-3278) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-3 (1556123236846010368)]} 0 2
[junit4] 2> 344796 INFO (qtp1775485046-3266) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 (1556123236846010368)]} 0 6
[junit4] 2> 344802 INFO (qtp2130827995-3285) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-3 (1556123236853350400)]} 0 3
[junit4] 2> 344803 INFO (qtp1775485046-3267) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 (1556123236853350400)]} 0 7
[junit4] 2> 344804 INFO (qtp2130827995-3283) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-1 (-1556123236856496128)]} 0 0
[junit4] 2> 344804 INFO (qtp1775485046-3266) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-1 (-1556123236856496128)]} 0 5
[junit4] 2> 344808 INFO (qtp2130827995-3279) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-1 (-1556123236863836160)]} 0 0
[junit4] 2> 344808 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-1 (-1556123236863836160)]} 0 2
[junit4] 2> 344810 INFO (qtp2130827995-3284) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-4 (1556123236863836161)]} 0 1
[junit4] 2> 344811 INFO (qtp1775485046-3270) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 (1556123236863836161)]} 0 4
[junit4] 2> 344815 INFO (qtp2130827995-3280) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-4 (1556123236869079040)]} 0 1
[junit4] 2> 344815 INFO (qtp1775485046-3271) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 (1556123236869079040)]} 0 4
[junit4] 2> 344817 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-2 (-1556123236872224768)]} 0 1
[junit4] 2> 344817 INFO (qtp1775485046-3273) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-2 (-1556123236872224768)]} 0 3
[junit4] 2> 344821 INFO (qtp2130827995-3278) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-2 (-1556123236876419072)]} 0 1
[junit4] 2> 344821 INFO (qtp1775485046-3268) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-2 (-1556123236876419072)]} 0 3
[junit4] 2> 344823 INFO (qtp2130827995-3285) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-5 (1556123236878516224)]} 0 1
[junit4] 2> 344824 INFO (qtp1775485046-3267) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 (1556123236878516224)]} 0 3
[junit4] 2> 344828 INFO (qtp2130827995-3283) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-5 (1556123236882710528)]} 0 1
[junit4] 2> 344828 INFO (qtp1775485046-3266) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 (1556123236882710528)]} 0 4
[junit4] 2> 344829 INFO (qtp2130827995-3279) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-6 (1556123236884807680)]} 0 1
[junit4] 2> 344830 INFO (qtp1775485046-3267) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-6 (1556123236884807680)]} 0 3
[junit4] 2> 344836 INFO (qtp2130827995-3280) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-6 (1556123236892147713)]} 0 1
[junit4] 2> 344836 INFO (qtp2130827995-3284) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-7 (1556123236892147712)]} 0 1
[junit4] 2> 344837 INFO (qtp1775485046-3270) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-7 (1556123236892147712)]} 0 4
[junit4] 2> 344838 INFO (qtp1775485046-3271) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 (1556123236892147713)]} 0 4
[junit4] 2> 344843 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-3 (-1556123236898439168)]} 0 1
[junit4] 2> 344844 INFO (qtp1775485046-3273) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-3 (-1556123236898439168)]} 0 5
[junit4] 2> 344845 INFO (qtp2130827995-3278) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-7 (1556123236899487744)]} 0 1
[junit4] 2> 344845 INFO (qtp1775485046-3268) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 (1556123236899487744)]} 0 5
[junit4] 2> 344851 INFO (qtp2130827995-3285) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-8 (1556123236906827776)]} 0 1
[junit4] 2> 344851 INFO (qtp2130827995-3283) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-3 (-1556123236906827777)]} 0 1
[junit4] 2> 344851 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-8 (1556123236906827776)]} 0 4
[junit4] 2> 344851 INFO (qtp1775485046-3266) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-3 (-1556123236906827777)]} 0 3
[junit4] 2> 344857 INFO (qtp2130827995-3279) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-8 (1556123236914167808)]} 0 1
[junit4] 2> 344857 INFO (qtp2130827995-3280) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-9 (1556123236914167809)]} 0 1
[junit4] 2> 344858 INFO (qtp1775485046-3270) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-9 (1556123236914167809)]} 0 4
[junit4] 2> 344858 INFO (qtp1775485046-3267) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 (1556123236914167808)]} 0 4
[junit4] 2> 344869 INFO (qtp2130827995-3284) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-10 (1556123236925702144)]} 0 1
[junit4] 2> 344869 INFO (qtp2130827995-3282) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-9 (1556123236925702145)]} 0 1
[junit4] 2> 344869 INFO (qtp1775485046-3273) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-10 (1556123236925702144)]} 0 4
[junit4] 2> 344869 INFO (qtp1775485046-3271) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 (1556123236925702145)]} 0 4
[junit4] 2> 344875 INFO (qtp2130827995-3278) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-4 (-1556123236933042177)]} 0 1
[junit4] 2> 344875 INFO (qtp1775485046-3272) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-4 (-1556123236933042177)]} 0 3
[junit4] 2> 344876 INFO (qtp2130827995-3285) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-10 (1556123236933042176)]} 0 2
[junit4] 2> 344876 INFO (qtp1775485046-3268) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 (1556123236933042176)]} 0 4
[junit4] 2> 344881 INFO (qtp2130827995-3279) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-4 (-1556123236939333633)]} 0 0
[junit4] 2> 344881 INFO (qtp2130827995-3283) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:35451/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-11 (1556123236939333632)]} 0 1
[junit4] 2> 344881 INFO (qtp1775485046-3266) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-11 (1556123236939333632)]} 0 3
[junit4] 2> 344881 INFO (qtp1775485046-3270) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-4 (-1556123236939333633)]} 0 3
[junit4] 2> 344885 INFO (TEST-HdfsRecoveryZkTest.test-seed#[8F6078DCBC564764]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@4b254359{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 344886 INFO (TEST-HdfsRecoveryZkTest.test-seed#[8F6078DCBC564764]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2057974568
[junit4] 2> 344887 ERROR (updateExecutor-362-thread-1-processing-http:////127.0.0.1:35132//solr//recoverytest_shard1_replica2 x:recoverytest_shard1_replica1 r:core_node1 n:127.0.0.1:35451_solr s:shard1 c:recoverytest) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.StreamingSolrClients error
[junit4] 2> java.net.ConnectException: Connection refused (Connection refused)
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 344911 WARN (qtp1775485046-3267) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.DistributedUpdateProcessor Error sending update to http://127.0.0.1:35132/solr
[junit4] 2> java.net.ConnectException: Connection refused (Connection refused)
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 344912 ERROR (qtp1775485046-3267) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on replica http://127.0.0.1:35132/solr/recoverytest_shard1_replica2/
[junit4] 2> java.net.ConnectException: Connection refused (Connection refused)
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 344912 INFO (qtp1775485046-3267) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.ZkController Put replica core=recoverytest_shard1_replica2 coreNodeName=core_node2 on 127.0.0.1:35132_solr into leader-initiated recovery.
[junit4] 2> 344912 INFO (qtp1775485046-3267) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-12 (1556123236944576512)]} 0 29
[junit4] 2> 344915 ERROR (updateExecutor-362-thread-2-processing-http:////127.0.0.1:35132//solr//recoverytest_shard1_replica2 x:recoverytest_shard1_replica1 r:core_node1 n:127.0.0.1:35451_solr s:shard1 c:recoverytest) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.StreamingSolrClients error
[junit4] 2> java.net.ConnectException: Connection refused (Connection refused)
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 344916 WARN (qtp1775485046-3273) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.DistributedUpdateProcessor Error sending update to http://127.0.0.1:35132/solr
[junit4] 2> java.net.ConnectException: Connection refused (Connection refused)
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 344916 ERROR (qtp1775485046-3273) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on replica http://127.0.0.1:35132/solr/recoverytest_shard1_replica2/
[junit4] 2> java.net.ConnectException: Connection refused (Connection refused)
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 344917 INFO (qtp1775485046-3273) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-11 (1556123236973936640)]} 0 29
[junit4] 2> 344918 INFO (updateExecutor-362-thread-1-processing-x:recoverytest_shard1_replica1 r:core_node1 http:////127.0.0.1:35132//solr//recoverytest_shard1_replica2// n:127.0.0.1:35451_solr s:shard1 c:recoverytest) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.LeaderInitiatedRecoveryThread Put replica core=recoverytest_shard1_replica2 coreNodeName=core_node2 on 127.0.0.1:35132_solr into leader-initiated recovery.
[junit4] 2> 344918 WARN (updateExecutor-362-thread-1-processing-x:recoverytest_shard1_replica1 r:core_node1 http:////127.0.0.1:35132//solr//recoverytest_shard1_replica2// n:127.0.0.1:35451_solr s:shard1 c:recoverytest) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.LeaderInitiatedRecoveryThread Leader is publishing core=recoverytest_shard1_replica2 coreNodeName =core_node2 state=down on behalf of un-reachable replica http://127.0.0.1:35132/solr/recoverytest_shard1_replica2/
[junit4] 2> 344919 INFO (updateExecutor-362-thread-1-processing-x:recoverytest_shard1_replica1 r:core_node1 http:////127.0.0.1:35132//solr//recoverytest_shard1_replica2// n:127.0.0.1:35451_solr s:shard1 c:recoverytest) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.LeaderInitiatedRecoveryThread LeaderInitiatedRecoveryThread-recoverytest_shard1_replica2 started running to send REQUESTRECOVERY command to http://127.0.0.1:35132/solr/recoverytest_shard1_replica2/; will try for a max of 600 secs
[junit4] 2> 344919 INFO (updateExecutor-362-thread-1-processing-x:recoverytest_shard1_replica1 r:core_node1 http:////127.0.0.1:35132//solr//recoverytest_shard1_replica2// n:127.0.0.1:35451_solr s:shard1 c:recoverytest) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.LeaderInitiatedRecoveryThread Asking core=recoverytest_shard1_replica2 coreNodeName=core_node2 on http://127.0.0.1:35132/solr to recover
[junit4] 2> 344921 ERROR (updateExecutor-362-thread-1-processing-x:recoverytest_shard1_replica1 r:core_node1 http:////127.0.0.1:35132//solr//recoverytest_shard1_replica2// n:127.0.0.1:35451_solr s:shard1 c:recoverytest) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.LeaderInitiatedRecoveryThread http://127.0.0.1:35132/solr: Could not tell a replica to recover:org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:35132/solr
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:617)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268)
[junit4] 2> at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
[junit4] 2> at org.apache.solr.cloud.LeaderInitiatedRecoveryThread.sendRecoveryCommandWithRetry(LeaderInitiatedRecoveryThread.java:208)
[junit4] 2> at org.apache.solr.cloud.LeaderInitiatedRecoveryThread.run(LeaderInitiatedRecoveryThread.java:95)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: java.net.ConnectException: Connection refused (Connection refused)
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:515)
[junit4] 2> ... 10 more
[junit4] 2>
[junit4] 2> 344924 INFO (coreCloseExecutor-1206-thread-1) [n:127.0.0.1:35132_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [recoverytest_shard1_replica2] CLOSING SolrCore org.apache.solr.core.SolrCore@1327219a
[junit4] 2> 344929 ERROR (updateExecutor-362-thread-2-processing-http:////127.0.0.1:35132//solr//recoverytest_shard1_replica2 x:recoverytest_shard1_replica1 r:core_node1 n:127.0.0.1:35451_solr s:shard1 c:recoverytest) [n:127.0.0.1:35451_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.StreamingSolrClients error
[junit4] 2> java.net.ConnectException: Connection refused (Connection refused)
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.
[...truncated too long message...]
] 2> at org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
[junit4] 2> at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
[junit4] 2> at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
[junit4] 2> at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
[junit4] 2> at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
[junit4] 2> at org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest.teardownClass(HdfsRecoveryZkTest.java:51)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> 362719 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:48378 48378
[junit4] 2> 362719 INFO (SUITE-HdfsRecoveryZkTest-seed#[8F6078DCBC564764]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {rnd_b=FST50, _version_=BlockTreeOrds(blocksize=128), a_t=BlockTreeOrds(blocksize=128), a_i=FST50, id=FST50}, docValues:{}, maxPointsInLeafNode=1776, maxMBSortInHeap=7.835301545866276, sim=RandomSimilarity(queryNorm=true,coord=no): {}, locale=is, timezone=Africa/Kinshasa
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=2,free=141807368,total=348127232
[junit4] 2> NOTE: All tests run in this JVM: [LeaderInitiatedRecoveryOnCommitTest, CollectionsAPIAsyncDistributedZkTest, ResponseLogComponentTest, TestSolrIndexConfig, SampleTest, TestReloadDeadlock, BooleanFieldTest, TestHdfsUpdateLog, BlockJoinFacetDistribTest, RuleEngineTest, NumericFieldsTest, TestCoreContainer, RecoveryAfterSoftCommitTest, BasicAuthIntegrationTest, DistributedSuggestComponentTest, SchemaVersionSpecificBehaviorTest, FileUtilsTest, TestHighFrequencyDictionaryFactory, SOLR749Test, RestartWhileUpdatingTest, TestCloudRecovery, DistributedQueryElevationComponentTest, TestBlobHandler, TestSlowCompositeReaderWrapper, MoreLikeThisHandlerTest, TestJsonRequest, TestLegacyFieldCache, TestSolrXml, ParsingFieldUpdateProcessorsTest, TestTolerantSearch, TestSchemaVersionResource, TestRTGBase, HdfsRecoveryZkTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=HdfsRecoveryZkTest -Dtests.seed=8F6078DCBC564764 -Dtests.slow=true -Dtests.locale=is -Dtests.timezone=Africa/Kinshasa -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | HdfsRecoveryZkTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
[junit4] > at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
[junit4] > at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
[junit4] > at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
[junit4] > at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:753)
[junit4] > at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:815)
[junit4] > at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1065)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:930)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:823)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:889)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$3(CoreContainer.java:541)
[junit4] > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([8F6078DCBC564764]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:269)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] Completed [69/679 (1!)] on J0 in 56.20s, 1 test, 1 failure <<< FAILURES!
[...truncated 64475 lines...]