You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2018/01/19 09:11:55 UTC
[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build #
1633 - Still unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1633/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC
1 tests failed.
FAILED: org.apache.solr.cloud.MoveReplicaHDFSTest.testFailedMove
Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:64676/solr/MoveReplicaHDFSTest_failed_coll_true, http://127.0.0.1:41716/solr/MoveReplicaHDFSTest_failed_coll_true]
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:64676/solr/MoveReplicaHDFSTest_failed_coll_true, http://127.0.0.1:41716/solr/MoveReplicaHDFSTest_failed_coll_true]
at __randomizedtesting.SeedInfo.seed([F7FCCB11519E25E2:5D3118E3E64DF032]:0)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:991)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:991)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.cloud.MoveReplicaTest.testFailedMove(MoveReplicaTest.java:309)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:64676/solr/MoveReplicaHDFSTest_failed_coll_true: no servers hosting shard: shard2
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:436)
... 47 more
Build Log:
[...truncated 13236 lines...]
[junit4] Suite: org.apache.solr.cloud.MoveReplicaHDFSTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/init-core-data-001
[junit4] 2> 2572781 WARN (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=12 numCloses=12
[junit4] 2> 2572781 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2572782 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 2572782 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 2572783 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-001
[junit4] 2> 2572783 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2572783 INFO (Thread-4608) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2572783 INFO (Thread-4608) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2572785 ERROR (Thread-4608) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2572883 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:64582
[junit4] 2> 2572886 INFO (zkConnectionManagerCallback-5243-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2572890 INFO (jetty-launcher-5240-thread-1) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T02:27:37+05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 2572891 INFO (jetty-launcher-5240-thread-3) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T02:27:37+05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 2572892 INFO (jetty-launcher-5240-thread-4) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T02:27:37+05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 2572892 INFO (jetty-launcher-5240-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2572892 INFO (jetty-launcher-5240-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2572892 INFO (jetty-launcher-5240-thread-3) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 2572890 INFO (jetty-launcher-5240-thread-2) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T02:27:37+05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 2572893 INFO (jetty-launcher-5240-thread-4) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2572893 INFO (jetty-launcher-5240-thread-4) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2572893 INFO (jetty-launcher-5240-thread-4) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 2572891 INFO (jetty-launcher-5240-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2572893 INFO (jetty-launcher-5240-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2572893 INFO (jetty-launcher-5240-thread-1) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 2572894 INFO (jetty-launcher-5240-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2572894 INFO (jetty-launcher-5240-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2572894 INFO (jetty-launcher-5240-thread-2) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 2572895 INFO (jetty-launcher-5240-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@dcd1278{/solr,null,AVAILABLE}
[junit4] 2> 2572895 INFO (jetty-launcher-5240-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7f3167a6{/solr,null,AVAILABLE}
[junit4] 2> 2572895 INFO (jetty-launcher-5240-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4796b014{/solr,null,AVAILABLE}
[junit4] 2> 2572895 INFO (jetty-launcher-5240-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3bb4fbcb{HTTP/1.1,[http/1.1]}{127.0.0.1:64676}
[junit4] 2> 2572895 INFO (jetty-launcher-5240-thread-3) [ ] o.e.j.s.Server Started @2578815ms
[junit4] 2> 2572895 INFO (jetty-launcher-5240-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=64676}
[junit4] 2> 2572895 ERROR (jetty-launcher-5240-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2572896 INFO (jetty-launcher-5240-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 2572895 INFO (jetty-launcher-5240-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2bb6305a{HTTP/1.1,[http/1.1]}{127.0.0.1:65486}
[junit4] 2> 2572896 INFO (jetty-launcher-5240-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2572896 INFO (jetty-launcher-5240-thread-2) [ ] o.e.j.s.Server Started @2578816ms
[junit4] 2> 2572896 INFO (jetty-launcher-5240-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=65486}
[junit4] 2> 2572896 ERROR (jetty-launcher-5240-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2572896 INFO (jetty-launcher-5240-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 2572896 INFO (jetty-launcher-5240-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2572896 INFO (jetty-launcher-5240-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2572897 INFO (jetty-launcher-5240-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-19T08:20:14.211Z
[junit4] 2> 2572897 INFO (jetty-launcher-5240-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2572897 INFO (jetty-launcher-5240-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-19T08:20:14.211Z
[junit4] 2> 2572897 INFO (jetty-launcher-5240-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7a879407{HTTP/1.1,[http/1.1]}{127.0.0.1:41716}
[junit4] 2> 2572897 INFO (jetty-launcher-5240-thread-1) [ ] o.e.j.s.Server Started @2578817ms
[junit4] 2> 2572897 INFO (jetty-launcher-5240-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41716}
[junit4] 2> 2572897 INFO (jetty-launcher-5240-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6b1a85e6{/solr,null,AVAILABLE}
[junit4] 2> 2572897 ERROR (jetty-launcher-5240-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2572898 INFO (jetty-launcher-5240-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 2572898 INFO (jetty-launcher-5240-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@ca035d5{HTTP/1.1,[http/1.1]}{127.0.0.1:46028}
[junit4] 2> 2572898 INFO (jetty-launcher-5240-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2572898 INFO (jetty-launcher-5240-thread-4) [ ] o.e.j.s.Server Started @2578818ms
[junit4] 2> 2572898 INFO (jetty-launcher-5240-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2572898 INFO (jetty-launcher-5240-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-19T08:20:14.212Z
[junit4] 2> 2572898 INFO (jetty-launcher-5240-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46028}
[junit4] 2> 2572898 ERROR (jetty-launcher-5240-thread-4) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2572898 INFO (jetty-launcher-5240-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 2572898 INFO (jetty-launcher-5240-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2572898 INFO (jetty-launcher-5240-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2572898 INFO (jetty-launcher-5240-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-19T08:20:14.212Z
[junit4] 2> 2572899 INFO (zkConnectionManagerCallback-5246-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2572900 INFO (zkConnectionManagerCallback-5249-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2572900 INFO (zkConnectionManagerCallback-5247-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2572900 INFO (jetty-launcher-5240-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2572900 INFO (zkConnectionManagerCallback-5251-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2572900 INFO (jetty-launcher-5240-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2572900 INFO (jetty-launcher-5240-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2572904 INFO (jetty-launcher-5240-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2572907 INFO (jetty-launcher-5240-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64582/solr
[junit4] 2> 2572907 INFO (jetty-launcher-5240-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64582/solr
[junit4] 2> 2572909 INFO (zkConnectionManagerCallback-5256-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2572910 INFO (jetty-launcher-5240-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64582/solr
[junit4] 2> 2572910 INFO (zkConnectionManagerCallback-5259-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2572911 INFO (zkConnectionManagerCallback-5263-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2572911 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x1610d8102930005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2572912 DEBUG (jetty-launcher-5240-thread-1) [n:127.0.0.1:41716_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$38/1286566746@501e6d71
[junit4] 2> 2572912 INFO (jetty-launcher-5240-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64582/solr
[junit4] 2> 2572913 DEBUG (jetty-launcher-5240-thread-2) [n:127.0.0.1:65486_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$38/1286566746@292ebd20
[junit4] 2> 2572914 DEBUG (jetty-launcher-5240-thread-3) [n:127.0.0.1:64676_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$38/1286566746@972c6c
[junit4] 2> 2572914 INFO (zkConnectionManagerCallback-5269-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2572914 INFO (zkConnectionManagerCallback-5266-thread-1-processing-n:127.0.0.1:41716_solr) [n:127.0.0.1:41716_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2572915 INFO (zkConnectionManagerCallback-5271-thread-1-processing-n:127.0.0.1:65486_solr) [n:127.0.0.1:65486_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2572915 INFO (zkConnectionManagerCallback-5273-thread-1-processing-n:127.0.0.1:64676_solr) [n:127.0.0.1:64676_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2572916 DEBUG (jetty-launcher-5240-thread-4) [n:127.0.0.1:46028_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$38/1286566746@1f7809a4
[junit4] 2> 2572918 INFO (zkConnectionManagerCallback-5275-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2573070 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 2573071 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573072 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 2573073 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2573073 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 2573074 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 2573074 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 2573075 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573075 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 2573076 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2573076 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 2573077 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 2573080 INFO (jetty-launcher-5240-thread-4) [n:127.0.0.1:46028_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2573080 INFO (jetty-launcher-5240-thread-3) [n:127.0.0.1:64676_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2573081 DEBUG (jetty-launcher-5240-thread-4) [n:127.0.0.1:46028_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/99375514523992076-127.0.0.1:46028_solr-n_0000000000
[junit4] 2> 2573081 DEBUG (jetty-launcher-5240-thread-3) [n:127.0.0.1:64676_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/99375514523992075-127.0.0.1:64676_solr-n_0000000001
[junit4] 2> 2573081 INFO (jetty-launcher-5240-thread-4) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46028_solr
[junit4] 2> 2573082 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/99375514523992076-127.0.0.1:46028_solr-n_0000000000
[junit4] 2> 2573082 DEBUG (jetty-launcher-5240-thread-3) [n:127.0.0.1:64676_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/99375514523992076-127.0.0.1:46028_solr-n_0000000000 to know if I could be the leader
[junit4] 2> 2573083 INFO (jetty-launcher-5240-thread-3) [n:127.0.0.1:64676_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64676_solr
[junit4] 2> 2573083 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2573083 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2573084 INFO (jetty-launcher-5240-thread-4) [n:127.0.0.1:46028_solr ] o.a.s.c.Overseer Overseer (id=99375514523992076-127.0.0.1:46028_solr-n_0000000000) starting
[junit4] 2> 2573084 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573084 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573085 INFO (zkCallback-5274-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2573086 INFO (zkCallback-5272-thread-1-processing-n:127.0.0.1:64676_solr) [n:127.0.0.1:64676_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2573088 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 2573088 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573088 INFO (jetty-launcher-5240-thread-2) [n:127.0.0.1:65486_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2573089 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 2573090 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2573090 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 2573091 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 2573093 INFO (jetty-launcher-5240-thread-2) [n:127.0.0.1:65486_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2573094 DEBUG (jetty-launcher-5240-thread-2) [n:127.0.0.1:65486_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/99375514523992074-127.0.0.1:65486_solr-n_0000000002
[junit4] 2> 2573095 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/99375514523992075-127.0.0.1:64676_solr-n_0000000001
[junit4] 2> 2573095 DEBUG (jetty-launcher-5240-thread-2) [n:127.0.0.1:65486_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/99375514523992075-127.0.0.1:64676_solr-n_0000000001 to know if I could be the leader
[junit4] 2> 2573095 DEBUG (jetty-launcher-5240-thread-2) [n:127.0.0.1:65486_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:65486_solr as DOWN
[junit4] 2> 2573096 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 2573096 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573097 INFO (jetty-launcher-5240-thread-2) [n:127.0.0.1:65486_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65486_solr
[junit4] 2> 2573097 INFO (jetty-launcher-5240-thread-1) [n:127.0.0.1:41716_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2573097 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 2573097 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2573098 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2573098 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2573098 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2573099 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2573099 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573099 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 2573100 INFO (zkCallback-5270-thread-1-processing-n:127.0.0.1:65486_solr) [n:127.0.0.1:65486_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2573100 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573101 INFO (zkCallback-5274-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2573101 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573101 DEBUG (jetty-launcher-5240-thread-4) [n:127.0.0.1:46028_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:46028_solr as DOWN
[junit4] 2> 2573101 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 2573101 DEBUG (OverseerAutoScalingTriggerThread-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 2573102 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 2573102 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573103 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/autoscaling.json
[junit4] 2> 2573104 DEBUG (OverseerAutoScalingTriggerThread-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 2573104 DEBUG (OverseerAutoScalingTriggerThread-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:65486_solr, 127.0.0.1:64676_solr]
[junit4] 2> 2573104 DEBUG (OverseerAutoScalingTriggerThread-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
[junit4] 2> 2573105 DEBUG (OverseerAutoScalingTriggerThread-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 2573105 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 2573105 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 2573105 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 2573105 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 2573105 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 2573105 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] 2> 2573105 INFO (jetty-launcher-5240-thread-4) [n:127.0.0.1:46028_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46028_solr
[junit4] 2> 2573106 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 2573106 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue-work
[junit4] 2> 2573107 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2573107 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2573107 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2573107 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2573109 INFO (zkCallback-5272-thread-1-processing-n:127.0.0.1:64676_solr) [n:127.0.0.1:64676_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2573109 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2573110 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573110 INFO (zkCallback-5274-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2573110 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573110 INFO (zkCallback-5270-thread-1-processing-n:127.0.0.1:65486_solr) [n:127.0.0.1:65486_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2573110 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573110 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573111 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:65486_solr"} current state version: 0
[junit4] 2> 2573111 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:65486_solr
[junit4] 2> 2573111 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:46028_solr"} current state version: 0
[junit4] 2> 2573111 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:46028_solr
[junit4] 2> 2573112 INFO (zkCallback-5272-thread-1-processing-n:127.0.0.1:64676_solr) [n:127.0.0.1:64676_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2573112 DEBUG (OverseerAutoScalingTriggerThread-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 2573112 DEBUG (OverseerAutoScalingTriggerThread-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 2573112 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 2573113 DEBUG (ScheduledTrigger-9655-thread-1) [n:127.0.0.1:46028_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 2573114 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 2573114 INFO (zkCallback-5265-thread-2-processing-n:127.0.0.1:41716_solr) [n:127.0.0.1:41716_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
[junit4] 2> 2573115 INFO (jetty-launcher-5240-thread-1) [n:127.0.0.1:41716_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2573115 DEBUG (jetty-launcher-5240-thread-1) [n:127.0.0.1:41716_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/99375514523992072-127.0.0.1:41716_solr-n_0000000003
[junit4] 2> 2573115 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/99375514523992074-127.0.0.1:65486_solr-n_0000000002
[junit4] 2> 2573115 DEBUG (jetty-launcher-5240-thread-1) [n:127.0.0.1:41716_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/99375514523992074-127.0.0.1:65486_solr-n_0000000002 to know if I could be the leader
[junit4] 2> 2573116 DEBUG (jetty-launcher-5240-thread-1) [n:127.0.0.1:41716_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:41716_solr as DOWN
[junit4] 2> 2573116 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2573116 INFO (jetty-launcher-5240-thread-1) [n:127.0.0.1:41716_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41716_solr
[junit4] 2> 2573117 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2573117 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2573117 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2573117 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2573124 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2573135 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573135 INFO (zkCallback-5265-thread-1-processing-n:127.0.0.1:41716_solr) [n:127.0.0.1:41716_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2573143 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573143 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573143 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573143 INFO (zkCallback-5274-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2573143 INFO (zkCallback-5272-thread-1-processing-n:127.0.0.1:64676_solr) [n:127.0.0.1:64676_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2573144 INFO (zkCallback-5270-thread-1-processing-n:127.0.0.1:65486_solr) [n:127.0.0.1:65486_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2573144 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.Overseer processMessage: queueSize: 3, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:41716_solr"} current state version: 0
[junit4] 2> 2573144 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:41716_solr
[junit4] 2> 2573211 INFO (jetty-launcher-5240-thread-1) [n:127.0.0.1:41716_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41716.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b5c506c
[junit4] 2> 2573216 INFO (jetty-launcher-5240-thread-1) [n:127.0.0.1:41716_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41716.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b5c506c
[junit4] 2> 2573217 INFO (jetty-launcher-5240-thread-1) [n:127.0.0.1:41716_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41716.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b5c506c
[junit4] 2> 2573217 INFO (jetty-launcher-5240-thread-1) [n:127.0.0.1:41716_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-001/node1/.
[junit4] 2> 2573251 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2573254 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2573303 INFO (jetty-launcher-5240-thread-3) [n:127.0.0.1:64676_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_64676.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b5c506c
[junit4] 2> 2573310 INFO (jetty-launcher-5240-thread-3) [n:127.0.0.1:64676_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_64676.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b5c506c
[junit4] 2> 2573310 INFO (jetty-launcher-5240-thread-3) [n:127.0.0.1:64676_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_64676.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b5c506c
[junit4] 2> 2573332 INFO (jetty-launcher-5240-thread-3) [n:127.0.0.1:64676_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-001/node3/.
[junit4] 2> 2573334 INFO (jetty-launcher-5240-thread-4) [n:127.0.0.1:46028_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46028.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b5c506c
[junit4] 2> 2573372 INFO (jetty-launcher-5240-thread-4) [n:127.0.0.1:46028_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46028.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b5c506c
[junit4] 2> 2573372 INFO (jetty-launcher-5240-thread-4) [n:127.0.0.1:46028_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46028.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b5c506c
[junit4] 2> 2573374 INFO (jetty-launcher-5240-thread-4) [n:127.0.0.1:46028_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-001/node4/.
[junit4] 2> 2573378 INFO (jetty-launcher-5240-thread-2) [n:127.0.0.1:65486_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_65486.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b5c506c
[junit4] 2> 2573384 INFO (jetty-launcher-5240-thread-2) [n:127.0.0.1:65486_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_65486.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b5c506c
[junit4] 2> 2573384 INFO (jetty-launcher-5240-thread-2) [n:127.0.0.1:65486_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_65486.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b5c506c
[junit4] 2> 2573385 INFO (jetty-launcher-5240-thread-2) [n:127.0.0.1:65486_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-001/node2/.
[junit4] 2> 2573461 INFO (zkConnectionManagerCallback-5285-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2573465 INFO (zkConnectionManagerCallback-5289-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2573466 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 2573466 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2573467 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2573467 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 2573467 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2573468 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 2573468 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64582/solr ready
[junit4] 2> 2573477 INFO (qtp1293670809-22463) [n:127.0.0.1:46028_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :overseerstatus with params action=OVERSEERSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2573477 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 2573478 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/overseer/collection-queue-work/qnr-0000000000
[junit4] 2> 2573478 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 2573478 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000000
[junit4] 2> 2573478 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 2573478 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 2573479 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000000
[junit4] 2> 2573479 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskQueue Returning topN elements: [/overseer/collection-queue-work/qn-0000000000, ]
[junit4] 2> 2573479 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ecf]]
[junit4] 2> 2573479 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as running
[junit4] 2> 2573479 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{"operation":"overseerstatus"}
[junit4] 2> 2573480 DEBUG (OverseerThreadFactory-9657-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000000
[junit4] 2> 2573480 DEBUG (OverseerThreadFactory-9657-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : overseerstatus , {"operation":"overseerstatus"}
[junit4] 2> 2573480 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1
[junit4] 2> 2573480 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 2573480 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 2573480 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 2573480 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 2573481 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 99 elements. ExcludeSet: [/overseer/collection-queue-work/qn-0000000000],[]
[junit4] 2> 2573483 DEBUG (OverseerThreadFactory-9657-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Completed task:[/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 2573484 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: data: /solr/overseer/collection-queue-work/qnr-0000000000
[junit4] 2> 2573484 DEBUG (zkCallback-5274-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskQueue NodeDataChanged fired on path /overseer/collection-queue-work/qnr-0000000000 state SyncConnected latchEventType null
[junit4] 2> 2573485 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 2573485 DEBUG (OverseerThreadFactory-9657-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as completed.
[junit4] 2> 2573485 DEBUG (OverseerThreadFactory-9657-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 2573485 DEBUG (OverseerThreadFactory-9657-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 2573485 DEBUG (OverseerThreadFactory-9657-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 2573485 DEBUG (OverseerThreadFactory-9657-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 2573485 DEBUG (OverseerThreadFactory-9657-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Message id:/overseer/collection-queue-work/qn-0000000000 complete, response:{leader=127.0.0.1:46028_solr,overseer_queue_size=0,overseer_work_queue_size=0,overseer_collection_queue_size=2,overseer_operations={am_i_leader={requests=3,errors=0,avgRequestsPerSecond=7.827264687604539,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=1.8647363333333333,medianRequestTime=2.087946,75thPcRequestTime=2.18311,95thPcRequestTime=2.18311,99thPcRequestTime=2.18311,999thPcRequestTime=2.18311},downnode={requests=3,errors=0,avgRequestsPerSecond=8.076263684162072,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=0.062914,medianRequestTime=0.049793,75thPcRequestTime=0.101578,95thPcRequestTime=0.101578,99thPcRequestTime=0.101578,999thPcRequestTime=0.101578}},collection_operations={am_i_leader={requests=3,errors=0,avgRequestsPerSecond=7.840069194987236,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=1.2235343333333333,medianRequestTime=0.902793,75thPcRequestTime=2.198343,95thPcRequestTime=2.198343,99thPcRequestTime=2.198343,999thPcRequestTime=2.198343},overseerstatus={requests=0,errors=0,avgRequestsPerSecond=0.0,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=0.0,medianRequestTime=0.0,75thPcRequestTime=0.0,95thPcRequestTime=0.0,99thPcRequestTime=0.0,999thPcRequestTime=0.0}},overseer_queue={},overseer_internal_queue={peek={avgRequestsPerSecond=2.6545496879247676,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=0.716968,medianRequestTime=0.716968,75thPcRequestTime=0.716968,95thPcRequestTime=0.716968,99thPcRequestTime=0.716968,999thPcRequestTime=0.716968}},collection_queue={peektopn_wait2000={avgRequestsPerSecond=2.6515415594629634,5minRateRequestsPerSecond=0.0,15minRateRequestsPerSecond=0.0,avgTimePerRequest=374.105027,medianRequestTime=374.105027,75thPcRequestTime=374.105027,95thPcRequestTime=374.105027,99thPcRequestTime=374.105027,999thPcRequestTime=374.105027}}}
[junit4] 2> 2573486 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 2573486 INFO (qtp1293670809-22463) [n:127.0.0.1:46028_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=OVERSEERSTATUS&wt=javabin&version=2} status=0 QTime=9
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 2573512 WARN (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 2573523 WARN (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2573527 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2573547 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_serv1.vm3_41206_hdfs____evz1wc/webapp
[junit4] 2> 2574020 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@serv1-vm3:41206
[junit4] 2> 2574113 DEBUG (ScheduledTrigger-9655-thread-1) [n:127.0.0.1:46028_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 2574126 WARN (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2574127 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2574137 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_63208_datanode____.revflo/webapp
[junit4] 2> 2574612 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:63208
[junit4] 2> 2574690 WARN (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2574696 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2574710 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_45127_datanode____g747ss/webapp
[junit4] 2> 2574763 ERROR (DataNode: [[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-002/hdfsBaseDir/data/data2/]] heartbeating to serv1-vm3/127.0.0.1:42007) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 2574793 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x2f318752bc78: from storage DS-d9ba6f14-c9d7-40d2-9c9d-7a41ea092d6e node DatanodeRegistration(127.0.0.1:42454, datanodeUuid=b8a54505-b67e-4638-ae2c-727ef138d48b, infoPort=43391, infoSecurePort=0, ipcPort=59141, storageInfo=lv=-56;cid=testClusterID;nsid=1165556706;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
[junit4] 2> 2574793 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x2f318752bc78: from storage DS-5127d20a-34f0-4483-8ebc-9fb7197df5a0 node DatanodeRegistration(127.0.0.1:42454, datanodeUuid=b8a54505-b67e-4638-ae2c-727ef138d48b, infoPort=43391, infoSecurePort=0, ipcPort=59141, storageInfo=lv=-56;cid=testClusterID;nsid=1165556706;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2575116 DEBUG (ScheduledTrigger-9655-thread-2) [n:127.0.0.1:46028_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 2575209 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45127
[junit4] 2> 2575306 ERROR (DataNode: [[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-002/hdfsBaseDir/data/data4/]] heartbeating to serv1-vm3/127.0.0.1:42007) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 2575313 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x2f31a665949e: from storage DS-5c61a442-de6d-43c9-8821-91cb960978d1 node DatanodeRegistration(127.0.0.1:46410, datanodeUuid=7b137dc3-0bb6-4d20-85b0-7831af19923d, infoPort=59030, infoSecurePort=0, ipcPort=64666, storageInfo=lv=-56;cid=testClusterID;nsid=1165556706;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2575313 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x2f31a665949e: from storage DS-e0b7dace-5777-44c9-be2b-1bbf06dcb91a node DatanodeRegistration(127.0.0.1:46410, datanodeUuid=7b137dc3-0bb6-4d20-85b0-7831af19923d, infoPort=59030, infoSecurePort=0, ipcPort=64666, storageInfo=lv=-56;cid=testClusterID;nsid=1165556706;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2575458 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testFailedMove
[junit4] 2> 2575459 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 2575460 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2575460 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2575460 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 2575460 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2575461 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 2575462 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T02:27:37+05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 2575463 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2575463 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2575463 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 2575463 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3a6a8c64{/solr,null,AVAILABLE}
[junit4] 2> 2575463 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@72956a40{HTTP/1.1,[http/1.1]}{127.0.0.1:64199}
[junit4] 2> 2575464 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.e.j.s.Server Started @2581384ms
[junit4] 2> 2575464 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=64199}
[junit4] 2> 2575464 ERROR (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2575464 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 2575464 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2575464 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2575464 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-19T08:20:16.778Z
[junit4] 2> 2575465 INFO (zkConnectionManagerCallback-5292-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2575466 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2575473 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64582/solr
[junit4] 2> 2575474 INFO (zkConnectionManagerCallback-5296-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2575476 DEBUG (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [n:127.0.0.1:64199_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$38/1286566746@7b7110d0
[junit4] 2> 2575476 INFO (zkConnectionManagerCallback-5298-thread-1-processing-n:127.0.0.1:64199_solr) [n:127.0.0.1:64199_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2575482 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
[junit4] 2> 2575482 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 2575483 INFO (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2575483 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 2575483 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 2575483 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 2575483 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 2575483 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] 2> 2575489 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 2575489 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2575489 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [n:127.0.0.1:64199_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2575489 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 2575490 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2575490 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 2575491 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 2575492 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [n:127.0.0.1:64199_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2575493 DEBUG (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [n:127.0.0.1:64199_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/99375514523992081-127.0.0.1:64199_solr-n_0000000004
[junit4] 2> 2575493 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/99375514523992072-127.0.0.1:41716_solr-n_0000000003
[junit4] 2> 2575493 DEBUG (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [n:127.0.0.1:64199_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/99375514523992072-127.0.0.1:41716_solr-n_0000000003 to know if I could be the leader
[junit4] 2> 2575494 DEBUG (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [n:127.0.0.1:64199_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:64199_solr as DOWN
[junit4] 2> 2575495 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2575495 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [n:127.0.0.1:64199_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64199_solr
[junit4] 2> 2575495 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2575496 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:64199_solr"} current state version: 0
[junit4] 2> 2575496 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:64199_solr
[junit4] 2> 2575496 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2575496 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2575496 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2575496 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2575496 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2575496 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 2575497 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2575497 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2575497 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2575497 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2575497 INFO (zkCallback-5265-thread-1-processing-n:127.0.0.1:41716_solr) [n:127.0.0.1:41716_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2575497 INFO (zkCallback-5270-thread-1-processing-n:127.0.0.1:65486_solr) [n:127.0.0.1:65486_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2575497 INFO (zkCallback-5274-thread-1-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2575497 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2575497 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2575497 INFO (zkCallback-5272-thread-1-processing-n:127.0.0.1:64676_solr) [n:127.0.0.1:64676_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2575498 INFO (zkCallback-5288-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2575498 INFO (zkCallback-5288-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2575498 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 2575499 INFO (zkCallback-5297-thread-1-processing-n:127.0.0.1:64199_solr) [n:127.0.0.1:64199_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 2575570 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [n:127.0.0.1:64199_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_64199.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b5c506c
[junit4] 2> 2575596 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2575596 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [n:127.0.0.1:64199_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_64199.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b5c506c
[junit4] 2> 2575597 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [n:127.0.0.1:64199_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_64199.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b5c506c
[junit4] 2> 2575597 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2575597 INFO (TEST-MoveReplicaHDFSTest.testFailedMove-seed#[F7FCCB11519E25E2]) [n:127.0.0.1:64199_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-001/node5/.
[junit4] 2> 2575639 INFO (zkConnectionManagerCallback-5302-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2575643 INFO (qtp1303830485-22445) [n:127.0.0.1:41716_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf1&autoAddReplicas=false&name=MoveReplicaHDFSTest_failed_coll_true&nrtReplicas=2&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2575643 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 2575644 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/overseer/collection-queue-work/qnr-0000000002
[junit4] 2> 2575644 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 2575644 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000002
[junit4] 2> 2575644 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 2575645 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 2575645 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000002
[junit4] 2> 2575645 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskQueue Returning topN elements: [/overseer/collection-queue-work/qn-0000000002, ]
[junit4] 2> 2575645 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ed1]]
[junit4] 2> 2575645 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000002] as running
[junit4] 2> 2575646 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000002 message:{
[junit4] 2> "name":"MoveReplicaHDFSTest_failed_coll_true",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "operation":"create"}
[junit4] 2> 2575646 DEBUG (OverseerThreadFactory-9657-thread-2-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000002
[junit4] 2> 2575646 DEBUG (OverseerThreadFactory-9657-thread-2-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
[junit4] 2> "name":"MoveReplicaHDFSTest_failed_coll_true",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "operation":"create"}
[junit4] 2> 2575646 INFO (OverseerThreadFactory-9657-thread-2-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection MoveReplicaHDFSTest_failed_coll_true
[junit4] 2> 2575646 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1
[junit4] 2> 2575646 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000002]
[junit4] 2> 2575646 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 2575646 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 2575646 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000002]
[junit4] 2> 2575646 DEBUG (OverseerCollectionConfigSetProcessor-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 99 elements. ExcludeSet: [/overseer/collection-queue-work/qn-0000000002],[]
[junit4] 2> 2575647 DEBUG (OverseerThreadFactory-9657-thread-2-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.a.c.Assign Identify nodes using default
[junit4] 2> 2575647 DEBUG (OverseerThreadFactory-9657-thread-2-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler creating collections conf node /collections/MoveReplicaHDFSTest_failed_coll_true
[junit4] 2> 2575649 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 2575649 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 2575649 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 2575649 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 2575649 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 2575649 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 2575649 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2575649 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2575649 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2575649 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2575650 DEBUG (OverseerThreadFactory-9657-thread-2-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.a.c.CreateCollectionCmd Check for collection zkNode:MoveReplicaHDFSTest_failed_coll_true
[junit4] 2> 2575650 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2575650 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2575650 DEBUG (OverseerThreadFactory-9657-thread-2-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.a.c.CreateCollectionCmd Collection zkNode exists
[junit4] 2> 2575650 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 2575651 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2575652 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2575653 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "name":"MoveReplicaHDFSTest_failed_coll_true",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "operation":"create"} current state version: 0
[junit4] 2> 2575653 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.o.ClusterStateMutator building a new cName: MoveReplicaHDFSTest_failed_coll_true
[junit4] 2> 2575653 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/MoveReplicaHDFSTest_failed_coll_true/state.json
[junit4] 2> 2575654 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2575654 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2575752 DEBUG (OverseerThreadFactory-9657-thread-2-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.a.c.CreateCollectionCmd Creating SolrCores for new collection MoveReplicaHDFSTest_failed_coll_true, shardNames [shard1, shard2] , message : {
[junit4] 2> "name":"MoveReplicaHDFSTest_failed_coll_true",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "operation":"create"}
[junit4] 2> 2575755 DEBUG (OverseerThreadFactory-9657-thread-2-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.a.c.CreateCollectionCmd Creating core MoveReplicaHDFSTest_failed_coll_true_shard1_replica_n1 as part of shard shard1 of collection MoveReplicaHDFSTest_failed_coll_true on 127.0.0.1:64676_solr
[junit4] 2> 2575757 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2575757 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2575758 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"MoveReplicaHDFSTest_failed_coll_true",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"MoveReplicaHDFSTest_failed_coll_true_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:64676/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"} current state version: 0
[junit4] 2> 2575758 INFO (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"MoveReplicaHDFSTest_failed_coll_true",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"MoveReplicaHDFSTest_failed_coll_true_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:64676/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2575759 DEBUG (OverseerThreadFactory-9657-thread-2-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.a.c.CreateCollectionCmd Creating core MoveReplicaHDFSTest_failed_coll_true_shard1_replica_n2 as part of shard shard1 of collection MoveReplicaHDFSTest_failed_coll_true on 127.0.0.1:41716_solr
[junit4] 2> 2575760 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.o.SliceMutator Old Slice: shard1:{
[junit4] 2> "range":"80000000-ffffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{}}
[junit4] 2> 2575760 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.o.SliceMutator New Slice: shard1:{
[junit4] 2> "range":"80000000-ffffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{"core_node3":{
[junit4] 2> "core":"MoveReplicaHDFSTest_failed_coll_true_shard1_replica_n1",
[junit4] 2> "base_url":"http://127.0.0.1:64676/solr",
[junit4] 2> "state":"down",
[junit4] 2> "node_name":null,
[junit4] 2> "type":"NRT"}}}
[junit4] 2> 2575760 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2575761 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2575762 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"MoveReplicaHDFSTest_failed_coll_true",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"MoveReplicaHDFSTest_failed_coll_true_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:41716/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"} current state version: 0
[junit4] 2> 2575762 INFO (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"MoveReplicaHDFSTest_failed_coll_true",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"MoveReplicaHDFSTest_failed_coll_true_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:41716/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2575763 DEBUG (OverseerThreadFactory-9657-thread-2-processing-n:127.0.0.1:46028_solr) [n:127.0.0.1:46028_solr ] o.a.s.c.a.c.CreateCollectionCmd Creating core MoveReplicaHDFSTest_failed_coll_true_shard2_replica_n4 as part of shard shard2 of collection MoveReplicaHDFSTest_failed_coll_true on 127.0.0.1:64199_solr
[junit4] 2> 2575763 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.o.SliceMutator Old Slice: shard1:{
[junit4] 2> "range":"80000000-ffffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{"core_node3":{
[junit4] 2> "core":"MoveReplicaHDFSTest_failed_coll_true_shard1_replica_n1",
[junit4] 2> "base_url":"http://127.0.0.1:64676/solr",
[junit4] 2> "state":"down",
[junit4] 2> "node_name":null,
[junit4] 2> "type":"NRT"}}}
[junit4] 2> 2575764 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.o.SliceMutator New Slice: shard1:{
[junit4] 2> "range":"80000000-ffffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{
[junit4] 2> "core_node3":{
[junit4] 2> "core":"MoveReplicaHDFSTest_failed_coll_true_shard1_replica_n1",
[junit4] 2> "base_url":"http://127.0.0.1:64676/solr",
[junit4] 2> "state":"down",
[junit4] 2> "node_name":null,
[junit4] 2> "type":"NRT"},
[junit4] 2> "core_node5":{
[junit4] 2> "core":"MoveReplicaHDFSTest_failed_coll_true_shard1_replica_n2",
[junit4] 2> "base_url":"http://127.0.0.1:41716/solr",
[junit4] 2> "state":"down",
[junit4] 2> "node_name":null,
[junit4] 2> "type":"NRT"}}}
[junit4] 2> 2575764 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 2575765 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 2575766 DEBUG (OverseerStateUpdate-99375514523992076-127.0.0.1:46028_solr-n_0000000000) [n:127.0.0.1:46028_solr ] o.a.s.c.Overseer processMessage: queueSize: 3, message = {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"MoveReplicaHDFSTest_failed_coll_true",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"MoveReplicaHDFSTest_failed_coll_true_shard2_replica_n4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:64199/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"} current state version: 0
[junit4] 2> 2575766 INFO (OverseerStateUpdate-9937551452
[...truncated too long message...]
olr-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 31 /solr/collections
[junit4] 2> 21 /solr/live_nodes
[junit4] 2> 2 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 2 /solr/overseer/collection-queue-work
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> 2677315 WARN (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 2677326 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 2677432 WARN (DataNode: [[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-002/hdfsBaseDir/data/data4/]] heartbeating to serv1-vm3/127.0.0.1:42007) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
[junit4] 2> 2677432 WARN (DataNode: [[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-002/hdfsBaseDir/data/data4/]] heartbeating to serv1-vm3/127.0.0.1:42007) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-175702623-127.0.0.1-1516350014808 (Datanode Uuid 7b137dc3-0bb6-4d20-85b0-7831af19923d) service to serv1-vm3/127.0.0.1:42007
[junit4] 2> 2677437 WARN (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 2677446 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
[junit4] 2> 2677459 WARN (DataNode: [[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-002/hdfsBaseDir/data/data2/]] heartbeating to serv1-vm3/127.0.0.1:42007) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
[junit4] 2> 2677460 WARN (DataNode: [[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001/tempDir-002/hdfsBaseDir/data/data2/]] heartbeating to serv1-vm3/127.0.0.1:42007) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-175702623-127.0.0.1-1516350014808 (Datanode Uuid b8a54505-b67e-4638-ae2c-727ef138d48b) service to serv1-vm3/127.0.0.1:42007
[junit4] 2> 2677485 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@serv1-vm3:0
[junit4] 2> 2677612 INFO (SUITE-MoveReplicaHDFSTest-seed#[F7FCCB11519E25E2]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:64582 64582
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.MoveReplicaHDFSTest_F7FCCB11519E25E2-001
[junit4] 2> Jan 19, 2018 8:21:59 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 26 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=Lucene50(blocksize=128)}, docValues:{_version_=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=2002, maxMBSortInHeap=7.284703015709914, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@76fc6ba8), locale=en-SG, timezone=Asia/Pontianak
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=3,free=89605368,total=536870912
[junit4] 2> NOTE: All tests run in this JVM: [TestFastOutputStream, TriLevelCompositeIdRoutingTest, HdfsLockFactoryTest, HighlighterTest, TestRTimerTree, OverseerTaskQueueTest, HdfsUnloadDistributedZkTest, SolrTestCaseJ4Test, TestReqParamsAPI, TestClusterProperties, RecoveryZkTest, TestRealTimeGet, TermsComponentTest, UniqFieldsUpdateProcessorFactoryTest, TestMultiValuedNumericRangeQuery, MBeansHandlerTest, SchemaVersionSpecificBehaviorTest, TestUniqueKeyFieldResource, TestSurroundQueryParser, HdfsDirectoryFactoryTest, SSLMigrationTest, TestShortCircuitedRequests, TestFieldSortValues, TestSolrJ, XmlUpdateRequestHandlerTest, TestTolerantSearch, TestMergePolicyConfig, ChaosMonkeyNothingIsSafeWithPullReplicasTest, RAMDirectoryFactoryTest, TestCursorMarkWithoutUniqueKey, TestLocalFSCloudBackupRestore, SolrCloudExampleTest, TestEmbeddedSolrServerAdminHandler, SuggestComponentContextFilterQueryTest, SmileWriterTest, PingRequestHandlerTest, TestLazyCores, QueryElevationComponentTest, StatsComponentTest, TestExclusionRuleCollectionAccess, TestFaceting, TestJavabinTupleStreamParser, NumericFieldsTest, CoreAdminCreateDiscoverTest, DeleteInactiveReplicaTest, DistribJoinFromCollectionTest, CdcrVersionReplicationTest, MetricTriggerTest, ResponseHeaderTest, TriggerIntegrationTest, OpenExchangeRatesOrgProviderTest, SolrMetricsIntegrationTest, DistributedFacetExistsSmallTest, DateRangeFieldTest, SharedFSAutoReplicaFailoverTest, BasicZkTest, TestClassicSimilarityFactory, TestCloudInspectUtil, TestLegacyField, SecurityConfHandlerTest, FieldMutatingUpdateProcessorTest, TestRecovery, TestSimpleTextCodec, SolrMetricManagerTest, TestPKIAuthenticationPlugin, SuggesterFSTTest, TestSchemaVersionResource, TestRemoteStreaming, TermVectorComponentDistributedTest, AtomicUpdateProcessorFactoryTest, TestBulkSchemaConcurrent, IndexSchemaRuntimeFieldTest, DistributedFacetSimpleRefinementLongTailTest, ConcurrentDeleteAndCreateCollectionTest, DirectoryFactoryTest, TestUtilizeNode, ZkCLITest, TestHalfAndHalfDocValues, TestSweetSpotSimilarityFactory, SpatialFilterTest, CoreMergeIndexesAdminHandlerTest, TestMultiWordSynonyms, TestReloadAndDeleteDocs, TestFoldingMultitermQuery, TestStressUserVersions, DateFieldTest, TestOverriddenPrefixQueryForCustomFieldType, TestExecutePlanAction, ZkFailoverTest, HdfsRestartWhileUpdatingTest, DistributedFacetPivotSmallTest, ForceLeaderTest, InfixSuggestersTest, CacheHeaderTest, TestReRankQParserPlugin, TestRetrieveFieldsOptimizer, MissingSegmentRecoveryTest, TestQueryUtils, DistributedQueryElevationComponentTest, SimpleCollectionCreateDeleteTest, TestCharFilters, TestConfigSetImmutable, TestCopyFieldCollectionResource, LeaderInitiatedRecoveryOnCommitTest, TestMacros, SOLR749Test, OutputWriterTest, DocumentBuilderTest, DistributedQueryComponentCustomSortTest, TestEmbeddedSolrServerSchemaAPI, WordBreakSolrSpellCheckerTest, TestOnReconnectListenerSupport, AddReplicaTest, DistribCursorPagingTest, OverseerRolesTest, DeleteShardTest, VersionInfoTest, LegacyCloudClusterPropTest, ReplaceNodeTest, BufferStoreTest, SolrCoreTest, CursorPagingTest, ConnectionManagerTest, TestSolrConfigHandlerCloud, SynonymTokenizerTest, SolrCloudReportersTest, LukeRequestHandlerTest, TestSolr4Spatial2, TestElisionMultitermQuery, TestRangeQuery, TestRawResponseWriter, TestPostingsSolrHighlighter, TestSimDistributedQueue, TestNodeLostTrigger, TestConfigReload, CloudExitableDirectoryReaderTest, TestCollationField, TestCloudPivotFacet, TestPhraseSuggestions, LeaderFailoverAfterPartitionTest, RestartWhileUpdatingTest, AnalyticsQueryTest, TestReload, TestBinaryField, TestSSLRandomization, TestSolrFieldCacheBean, TestGroupingSearch, TestManagedSchemaAPI, TestConfigSetsAPIExclusivity, DistributedDebugComponentTest, TestCollectionsAPIViaSolrCloudCluster, MultiThreadedOCPTest, CollectionReloadTest, StatelessScriptUpdateProcessorFactoryTest, TestInitQParser, HdfsBasicDistributedZkTest, PluginInfoTest, NodeLostTriggerTest, TestXIncludeConfig, TestCryptoKeys, HdfsThreadLeakTest, SolrPluginUtilsTest, TestNumericTerms64, SolrGraphiteReporterTest, TestSha256AuthenticationProvider, PKIAuthenticationIntegrationTest, TestSolrXml, SearchHandlerTest, TestAuthenticationFramework, MetricsHandlerTest, TestBlendedInfixSuggestions, TestPointFields, TestDistributedStatsComponentCardinality, TestMaxTokenLenTokenizer, ConvertedLegacyTest, TestSchemalessBufferedUpdates, TestLegacyFieldCache, TestRebalanceLeaders, TestCloudSchemaless, DistributedMLTComponentTest, TestRestManager, TestNonDefinedSimilarityFactory, RulesTest, BigEndianAscendingWordSerializerTest, TestTrackingShardHandlerFactory, TestSuggestSpellingConverter, BasicDistributedZkTest, OpenCloseCoreStressTest, LeaderElectionIntegrationTest, TestReplicationHandler, ShardRoutingCustomTest, ZkControllerTest, TestJoin, TestCoreContainer, SimpleFacetsTest, SolrCmdDistributorTest, TestFiltering, BasicFunctionalityTest, SoftAutoCommitTest, AssignBackwardCompatibilityTest, CreateCollectionCleanupTest, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, LeaderElectionContextKeyTest, LeaderInitiatedRecoveryOnShardRestartTest, MigrateRouteKeyTest, MoveReplicaHDFSTest]
[junit4] Completed [522/768 (1!)] on J0 in 114.75s, 4 tests, 1 error <<< FAILURES!
[...truncated 49025 lines...]
[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build #
1634 - Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1634/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
3 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
Error Message:
3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 1) Thread[id=16653, name=jetty-launcher-3081-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) 2) Thread[id=16719, name=jetty-launcher-3081-thread-1-SendThread(127.0.0.1:46985), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 3) Thread[id=16720, name=jetty-launcher-3081-thread-1-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
1) Thread[id=16653, name=jetty-launcher-3081-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
2) Thread[id=16719, name=jetty-launcher-3081-thread-1-SendThread(127.0.0.1:46985), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
3) Thread[id=16720, name=jetty-launcher-3081-thread-1-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
at __randomizedtesting.SeedInfo.seed([694F82930B56CC0C]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=16719, name=jetty-launcher-3081-thread-1-SendThread(127.0.0.1:46985), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=16719, name=jetty-launcher-3081-thread-1-SendThread(127.0.0.1:46985), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
at __randomizedtesting.SeedInfo.seed([694F82930B56CC0C]:0)
FAILED: org.apache.solr.cloud.TestStressInPlaceUpdates.stressTest
Error Message:
Captured an uncaught exception in thread: Thread[id=14999, name=WRITER9, state=RUNNABLE, group=TGRP-TestStressInPlaceUpdates]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=14999, name=WRITER9, state=RUNNABLE, group=TGRP-TestStressInPlaceUpdates]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:44773/m/z/collection1
at __randomizedtesting.SeedInfo.seed([694F82930B56CC0C]:0)
at org.apache.solr.cloud.TestStressInPlaceUpdates$1.run(TestStressInPlaceUpdates.java:306)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:44773/m/z/collection1
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at org.apache.solr.cloud.TestStressInPlaceUpdates.deleteDocAndGetVersion(TestStressInPlaceUpdates.java:570)
at org.apache.solr.cloud.TestStressInPlaceUpdates$1.run(TestStressInPlaceUpdates.java:218)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
at sun.security.ssl.InputRecord.read(InputRecord.java:503)
at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:940)
at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
... 6 more
Build Log:
[...truncated 12530 lines...]
[junit4] Suite: org.apache.solr.cloud.TestStressInPlaceUpdates
[junit4] 2> 1815488 INFO (SUITE-TestStressInPlaceUpdates-seed#[694F82930B56CC0C]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/init-core-data-001
[junit4] 2> 1815488 WARN (SUITE-TestStressInPlaceUpdates-seed#[694F82930B56CC0C]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=6 numCloses=6
[junit4] 2> 1815488 INFO (SUITE-TestStressInPlaceUpdates-seed#[694F82930B56CC0C]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 1815490 INFO (SUITE-TestStressInPlaceUpdates-seed#[694F82930B56CC0C]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 1815490 INFO (SUITE-TestStressInPlaceUpdates-seed#[694F82930B56CC0C]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /m/z
[junit4] 2> 1815490 INFO (SUITE-TestStressInPlaceUpdates-seed#[694F82930B56CC0C]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore
[junit4] 2> 1815491 INFO (SUITE-TestStressInPlaceUpdates-seed#[694F82930B56CC0C]-worker) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 1815508 INFO (SUITE-TestStressInPlaceUpdates-seed#[694F82930B56CC0C]-worker) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1815515 INFO (SUITE-TestStressInPlaceUpdates-seed#[694F82930B56CC0C]-worker) [ ] o.a.s.s.IndexSchema [null] Schema name=inplace-updates
[junit4] 2> 1815517 INFO (SUITE-TestStressInPlaceUpdates-seed#[694F82930B56CC0C]-worker) [ ] o.a.s.s.IndexSchema Loaded schema inplace-updates/1.6 with uniqueid field id
[junit4] 2> 1815542 INFO (SUITE-TestStressInPlaceUpdates-seed#[694F82930B56CC0C]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1815546 INFO (SUITE-TestStressInPlaceUpdates-seed#[694F82930B56CC0C]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1815546 INFO (SUITE-TestStressInPlaceUpdates-seed#[694F82930B56CC0C]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1815547 INFO (coreLoadExecutor-4940-thread-1) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1815547 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 1815565 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1815571 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=inplace-updates
[junit4] 2> 1815579 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema inplace-updates/1.6 with uniqueid field id
[junit4] 2> 1815580 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1, trusted=true
[junit4] 2> 1815580 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1815580 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1815580 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/init-core-data-001/]
[junit4] 2> 1815581 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=63.177734375, floorSegmentMB=1.435546875, forceMergeDeletesPctAllowed=4.527407614852849, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1815583 WARN (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1815614 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1815614 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1815614 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1815614 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1815615 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=1.84375, floorSegmentMB=0.29296875, forceMergeDeletesPctAllowed=3.523379920174202, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6691715687630362
[junit4] 2> 1815615 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@35f63757[collection1] main]
[junit4] 2> 1815615 WARN (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
[junit4] 2> 1815616 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1815616 INFO (coreLoadExecutor-4940-thread-1) [ x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1590045919917637632
[junit4] 2> 1815616 INFO (searcherExecutor-4941-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@35f63757[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1815618 INFO (SUITE-TestStressInPlaceUpdates-seed#[694F82930B56CC0C]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore end
[junit4] 2> 1815620 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1815620 INFO (Thread-5200) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1815620 INFO (Thread-5200) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1815622 ERROR (Thread-5200) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1815720 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.ZkTestServer start zk server on port:40228
[junit4] 2> 1815725 INFO (zkConnectionManagerCallback-2715-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1815734 INFO (zkConnectionManagerCallback-2717-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1815753 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1815760 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/schema-inplace-updates.xml to /configs/conf1/schema.xml
[junit4] 2> 1815765 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1815769 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1815773 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1815778 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1815782 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1815786 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1815789 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1815792 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1815794 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1815799 INFO (zkConnectionManagerCallback-2720-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1815801 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 1815913 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T04:27:37+07:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1815913 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1815913 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1815913 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 1815914 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@45644cf0{/m/z,null,AVAILABLE}
[junit4] 2> 1815914 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@73a6d109{SSL,[ssl, http/1.1]}{127.0.0.1:59055}
[junit4] 2> 1815914 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.Server Started @1819752ms
[junit4] 2> 1815914 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/tempDir-001/control/data, hostContext=/m/z, hostPort=59055, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/control-001/cores}
[junit4] 2> 1815914 ERROR (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1815914 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 1815914 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1815914 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1815914 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-19T18:19:16.030Z
[junit4] 2> 1815916 INFO (zkConnectionManagerCallback-2722-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1815917 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1815917 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/control-001/solr.xml
[junit4] 2> 1815920 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1815920 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1815920 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1815922 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40228/solr
[junit4] 2> 1815923 INFO (zkConnectionManagerCallback-2726-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1815926 INFO (zkConnectionManagerCallback-2728-thread-1-processing-n:127.0.0.1:59055_m%2Fz) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1816029 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1816029 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59055_m%2Fz
[junit4] 2> 1816030 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.Overseer Overseer (id=99377869995245573-127.0.0.1:59055_m%2Fz-n_0000000000) starting
[junit4] 2> 1816038 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59055_m%2Fz
[junit4] 2> 1816041 INFO (OverseerStateUpdate-99377869995245573-127.0.0.1:59055_m%2Fz-n_0000000000) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1816093 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:59055_m%2Fz ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1816099 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:59055_m%2Fz ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1816099 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:59055_m%2Fz ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1816100 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/control-001/cores
[junit4] 2> 1816117 INFO (zkConnectionManagerCallback-2734-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1816119 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1816120 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40228/solr ready
[junit4] 2> 1816134 INFO (qtp827087756-14841) [n:127.0.0.1:59055_m%2Fz ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:59055_m%252Fz&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1816137 INFO (OverseerThreadFactory-4953-thread-1-processing-n:127.0.0.1:59055_m%2Fz) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 1816256 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1816257 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1816368 INFO (zkCallback-2727-thread-1-processing-n:127.0.0.1:59055_m%2Fz) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1817291 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1817315 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=inplace-updates
[junit4] 2> 1817318 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema inplace-updates/1.6 with uniqueid field id
[junit4] 2> 1817318 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 1817319 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1817319 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1817319 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 1817321 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=63.177734375, floorSegmentMB=1.435546875, forceMergeDeletesPctAllowed=4.527407614852849, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1817324 WARN (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1817365 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1817365 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1817365 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1817365 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1817366 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=1.84375, floorSegmentMB=0.29296875, forceMergeDeletesPctAllowed=3.523379920174202, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6691715687630362
[junit4] 2> 1817366 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@17e9c7a1[control_collection_shard1_replica_n1] main]
[junit4] 2> 1817368 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1817369 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1817369 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1817369 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1590045921755791360
[junit4] 2> 1817370 INFO (searcherExecutor-4956-thread-1-processing-n:127.0.0.1:59055_m%2Fz x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@17e9c7a1[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1817375 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1817375 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1817375 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:59055/m/z/control_collection_shard1_replica_n1/
[junit4] 2> 1817375 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1817375 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:59055/m/z/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1817375 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1817380 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:59055/m/z/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 1817483 INFO (zkCallback-2727-thread-2-processing-n:127.0.0.1:59055_m%2Fz) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1817531 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1817535 INFO (qtp827087756-14843) [n:127.0.0.1:59055_m%2Fz c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1279
[junit4] 2> 1817544 INFO (qtp827087756-14841) [n:127.0.0.1:59055_m%2Fz ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1817637 INFO (zkCallback-2727-thread-2-processing-n:127.0.0.1:59055_m%2Fz) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1818143 INFO (OverseerCollectionConfigSetProcessor-99377869995245573-127.0.0.1:59055_m%2Fz-n_0000000000) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1818545 INFO (qtp827087756-14841) [n:127.0.0.1:59055_m%2Fz ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:59055_m%252Fz&wt=javabin&version=2} status=0 QTime=2411
[junit4] 2> 1818553 INFO (zkConnectionManagerCallback-2738-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1818557 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1818561 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40228/solr ready
[junit4] 2> 1818561 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1818565 INFO (qtp827087756-14838) [n:127.0.0.1:59055_m%2Fz ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1818571 INFO (OverseerThreadFactory-4953-thread-2-processing-n:127.0.0.1:59055_m%2Fz) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1818574 WARN (OverseerThreadFactory-4953-thread-2-processing-n:127.0.0.1:59055_m%2Fz) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 1818786 INFO (qtp827087756-14838) [n:127.0.0.1:59055_m%2Fz ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1818786 INFO (qtp827087756-14838) [n:127.0.0.1:59055_m%2Fz ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=221
[junit4] 2> 1818924 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-1-001 of type NRT
[junit4] 2> 1818926 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T04:27:37+07:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1818927 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1818927 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1818927 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 1818927 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7c684fc5{/m/z,null,AVAILABLE}
[junit4] 2> 1818928 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@79eb1c53{SSL,[ssl, http/1.1]}{127.0.0.1:44773}
[junit4] 2> 1818928 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.Server Started @1822767ms
[junit4] 2> 1818928 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/m/z, hostPort=44773, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-1-001/cores}
[junit4] 2> 1818928 ERROR (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1818928 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 1818928 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1818928 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1818928 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-19T18:19:19.044Z
[junit4] 2> 1818929 INFO (zkConnectionManagerCallback-2740-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1818931 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1818931 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-1-001/solr.xml
[junit4] 2> 1818934 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1818934 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1818934 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1818936 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40228/solr
[junit4] 2> 1818937 INFO (zkConnectionManagerCallback-2744-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1818940 INFO (zkConnectionManagerCallback-2746-thread-1-processing-n:127.0.0.1:44773_m%2Fz) [n:127.0.0.1:44773_m%2Fz ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1818947 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:44773_m%2Fz ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1818950 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:44773_m%2Fz ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1818952 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:44773_m%2Fz ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44773_m%2Fz
[junit4] 2> 1818953 INFO (zkCallback-2737-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1818953 INFO (zkCallback-2727-thread-2-processing-n:127.0.0.1:59055_m%2Fz) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1818954 INFO (zkCallback-2745-thread-1-processing-n:127.0.0.1:44773_m%2Fz) [n:127.0.0.1:44773_m%2Fz ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1819002 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:44773_m%2Fz ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1819009 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:44773_m%2Fz ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1819009 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:44773_m%2Fz ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1819010 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:44773_m%2Fz ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-1-001/cores
[junit4] 2> 1819050 INFO (qtp1039936643-14888) [n:127.0.0.1:44773_m%2Fz ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:44773_m%252Fz&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1819054 INFO (OverseerCollectionConfigSetProcessor-99377869995245573-127.0.0.1:59055_m%2Fz-n_0000000000) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1819054 INFO (OverseerThreadFactory-4953-thread-3-processing-n:127.0.0.1:59055_m%2Fz) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:44773_m%2Fz for creating new replica
[junit4] 2> 1819061 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1819061 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1820089 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1820119 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema [collection1_shard1_replica_n21] Schema name=inplace-updates
[junit4] 2> 1820122 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema Loaded schema inplace-updates/1.6 with uniqueid field id
[junit4] 2> 1820122 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n21' using configuration from collection collection1, trusted=true
[junit4] 2> 1820123 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n21' (registry 'solr.core.collection1.shard1.replica_n21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1820123 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1820123 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [[collection1_shard1_replica_n21] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-1-001/cores/collection1_shard1_replica_n21], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-1-001/cores/collection1_shard1_replica_n21/data/]
[junit4] 2> 1820124 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=63.177734375, floorSegmentMB=1.435546875, forceMergeDeletesPctAllowed=4.527407614852849, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1820127 WARN (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1820159 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1820159 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1820160 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1820160 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1820161 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=1.84375, floorSegmentMB=0.29296875, forceMergeDeletesPctAllowed=3.523379920174202, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6691715687630362
[junit4] 2> 1820161 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.SolrIndexSearcher Opening [Searcher@2616ca8b[collection1_shard1_replica_n21] main]
[junit4] 2> 1820162 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1820163 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1820163 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1820163 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1590045924685512704
[junit4] 2> 1820164 INFO (searcherExecutor-4967-thread-1-processing-n:127.0.0.1:44773_m%2Fz x:collection1_shard1_replica_n21 s:shard1 c:collection1) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [collection1_shard1_replica_n21] Registered new searcher Searcher@2616ca8b[collection1_shard1_replica_n21] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1820169 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1820169 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1820169 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:44773/m/z/collection1_shard1_replica_n21/
[junit4] 2> 1820170 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1820170 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy https://127.0.0.1:44773/m/z/collection1_shard1_replica_n21/ has no replicas
[junit4] 2> 1820170 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1820174 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:44773/m/z/collection1_shard1_replica_n21/ shard1
[junit4] 2> 1820330 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1820335 INFO (qtp1039936643-14893) [n:127.0.0.1:44773_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1274
[junit4] 2> 1820340 INFO (qtp1039936643-14888) [n:127.0.0.1:44773_m%2Fz ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:44773_m%252Fz&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1290
[junit4] 2> 1820488 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-2-001 of type NRT
[junit4] 2> 1820489 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T04:27:37+07:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1820490 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1820490 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1820490 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 1820490 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2759e497{/m/z,null,AVAILABLE}
[junit4] 2> 1820491 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5a856979{SSL,[ssl, http/1.1]}{127.0.0.1:52246}
[junit4] 2> 1820491 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.Server Started @1824331ms
[junit4] 2> 1820491 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/m/z, hostPort=52246, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-2-001/cores}
[junit4] 2> 1820491 ERROR (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1820491 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 1820491 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1820491 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1820491 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-19T18:19:20.607Z
[junit4] 2> 1820493 INFO (zkConnectionManagerCallback-2750-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1820494 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1820494 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-2-001/solr.xml
[junit4] 2> 1820497 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1820497 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1820498 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1820500 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40228/solr
[junit4] 2> 1820501 INFO (zkConnectionManagerCallback-2754-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1820504 INFO (zkConnectionManagerCallback-2756-thread-1-processing-n:127.0.0.1:52246_m%2Fz) [n:127.0.0.1:52246_m%2Fz ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1820513 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:52246_m%2Fz ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1820516 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:52246_m%2Fz ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1820519 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:52246_m%2Fz ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52246_m%2Fz
[junit4] 2> 1820520 INFO (zkCallback-2737-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1820520 INFO (zkCallback-2727-thread-1-processing-n:127.0.0.1:59055_m%2Fz) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1820521 INFO (zkCallback-2745-thread-1-processing-n:127.0.0.1:44773_m%2Fz) [n:127.0.0.1:44773_m%2Fz ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1820522 INFO (zkCallback-2755-thread-1-processing-n:127.0.0.1:52246_m%2Fz) [n:127.0.0.1:52246_m%2Fz ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1820622 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:52246_m%2Fz ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1820628 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:52246_m%2Fz ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1820628 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:52246_m%2Fz ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1820629 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:52246_m%2Fz ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-2-001/cores
[junit4] 2> 1820656 INFO (qtp1039936643-14890) [n:127.0.0.1:44773_m%2Fz ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:52246_m%252Fz&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1820659 INFO (OverseerCollectionConfigSetProcessor-99377869995245573-127.0.0.1:59055_m%2Fz-n_0000000000) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1820660 INFO (OverseerThreadFactory-4953-thread-4-processing-n:127.0.0.1:59055_m%2Fz) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:52246_m%2Fz for creating new replica
[junit4] 2> 1820676 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1820677 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1821706 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 1821735 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.IndexSchema [collection1_shard1_replica_n23] Schema name=inplace-updates
[junit4] 2> 1821737 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.IndexSchema Loaded schema inplace-updates/1.6 with uniqueid field id
[junit4] 2> 1821737 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n23' using configuration from collection collection1, trusted=true
[junit4] 2> 1821738 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n23' (registry 'solr.core.collection1.shard1.replica_n23') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1821738 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1821738 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrCore [[collection1_shard1_replica_n23] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-2-001/cores/collection1_shard1_replica_n23], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-2-001/cores/collection1_shard1_replica_n23/data/]
[junit4] 2> 1821739 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=63.177734375, floorSegmentMB=1.435546875, forceMergeDeletesPctAllowed=4.527407614852849, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1821741 WARN (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1821773 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1821773 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1821774 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1821774 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1821775 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=1.84375, floorSegmentMB=0.29296875, forceMergeDeletesPctAllowed=3.523379920174202, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6691715687630362
[junit4] 2> 1821775 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.SolrIndexSearcher Opening [Searcher@2bd45d1b[collection1_shard1_replica_n23] main]
[junit4] 2> 1821777 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1821777 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1821777 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1821778 INFO (searcherExecutor-4978-thread-1-processing-n:127.0.0.1:52246_m%2Fz x:collection1_shard1_replica_n23 s:shard1 c:collection1) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrCore [collection1_shard1_replica_n23] Registered new searcher Searcher@2bd45d1b[collection1_shard1_replica_n23] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1821778 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1590045926378962944
[junit4] 2> 1821783 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.ZkController Core needs to recover:collection1_shard1_replica_n23
[junit4] 2> 1821783 INFO (updateExecutor-2751-thread-1-processing-n:127.0.0.1:52246_m%2Fz x:collection1_shard1_replica_n23 s:shard1 c:collection1) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 1821784 INFO (recoveryExecutor-2752-thread-1-processing-n:127.0.0.1:52246_m%2Fz x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 1821784 INFO (recoveryExecutor-2752-thread-1-processing-n:127.0.0.1:52246_m%2Fz x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 1821784 INFO (qtp1577132256-14922) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1108
[junit4] 2> 1821784 INFO (recoveryExecutor-2752-thread-1-processing-n:127.0.0.1:52246_m%2Fz x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard1_replica_n23]
[junit4] 2> 1821784 INFO (recoveryExecutor-2752-thread-1-processing-n:127.0.0.1:52246_m%2Fz x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 1821784 INFO (recoveryExecutor-2752-thread-1-processing-n:127.0.0.1:52246_m%2Fz x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard1_replica_n23] as recovering, leader is [https://127.0.0.1:44773/m/z/collection1_shard1_replica_n21/] and I am [https://127.0.0.1:52246/m/z/collection1_shard1_replica_n23/]
[junit4] 2> 1821787 INFO (recoveryExecutor-2752-thread-1-processing-n:127.0.0.1:52246_m%2Fz x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:52246_m%2Fz c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Sending prep recovery command to [https://127.0.0.1:44773/m/z]; [WaitForState: action=PREPRECOVERY&core=collection1_shard1_replica_n21&nodeName=127.0.0.1:52246_m%252Fz&coreNodeName=core_node24&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 1821787 INFO (qtp1039936643-14890) [n:127.0.0.1:44773_m%2Fz ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:52246_m%252Fz&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1131
[junit4] 2> 1821790 INFO (qtp1039936643-14888) [n:127.0.0.1:44773_m%2Fz ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node24, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
[junit4] 2> 1821790 INFO (qtp1039936643-14888) [n:127.0.0.1:44773_m%2Fz ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_n21, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:52246_m%2Fz, coreNodeName=core_node24, onlyIfActiveCheckResult=false, nodeProps: core_node24:{"core":"collection1_shard1_replica_n23","base_url":"https://127.0.0.1:52246/m/z","node_name":"127.0.0.1:52246_m%2Fz","state":"down","type":"NRT"}
[junit4] 2> 1821901 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-3-001 of type NRT
[junit4] 2> 1821902 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T04:27:37+07:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1821903 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1821903 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1821903 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 1821903 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7b10cb{/m/z,null,AVAILABLE}
[junit4] 2> 1821904 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@f4a32c9{SSL,[ssl, http/1.1]}{127.0.0.1:60163}
[junit4] 2> 1821904 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.e.j.s.Server Started @1825744ms
[junit4] 2> 1821904 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/m/z, hostPort=60163, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-3-001/cores}
[junit4] 2> 1821904 ERROR (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1821904 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 1821904 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1821904 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1821904 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-19T18:19:22.020Z
[junit4] 2> 1821906 INFO (zkConnectionManagerCallback-2761-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1821907 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1821907 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-3-001/solr.xml
[junit4] 2> 1821910 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1821910 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1821910 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1821912 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40228/solr
[junit4] 2> 1821913 INFO (zkConnectionManagerCallback-2765-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1821916 INFO (zkConnectionManagerCallback-2767-thread-1-processing-n:127.0.0.1:60163_m%2Fz) [n:127.0.0.1:60163_m%2Fz ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1821925 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:60163_m%2Fz ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1821928 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:60163_m%2Fz ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1821930 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:60163_m%2Fz ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60163_m%2Fz
[junit4] 2> 1821931 INFO (zkCallback-2727-thread-2-processing-n:127.0.0.1:59055_m%2Fz) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1821931 INFO (zkCallback-2745-thread-1-processing-n:127.0.0.1:44773_m%2Fz) [n:127.0.0.1:44773_m%2Fz ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1821931 INFO (zkCallback-2737-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1821932 INFO (zkCallback-2755-thread-1-processing-n:127.0.0.1:52246_m%2Fz) [n:127.0.0.1:52246_m%2Fz ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1821933 INFO (zkCallback-2766-thread-1-processing-n:127.0.0.1:60163_m%2Fz) [n:127.0.0.1:60163_m%2Fz ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1821987 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:60163_m%2Fz ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1821993 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:60163_m%2Fz ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1821993 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:60163_m%2Fz ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@635c1dd6
[junit4] 2> 1821994 INFO (TEST-TestStressInPlaceUpdates.stressTest-seed#[694F82930B56CC0C]) [n:127.0.0.1:60163_m%2Fz ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestStressInPlaceUpdates_694F82930B56CC0C-001/shard-3-001/cores
[junit4] 2> 1822025 INFO (qtp1039936643-14890) [n:127.0.0.1:44773_m%2Fz ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:60163_m%252Fz&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1822029 INFO (OverseerCollectionConfigSetProcessor-99377869995245573-127.0.0.1:59055_m%2Fz-n_0000000000) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1822029 INFO (OverseerThreadFactory-4953-thread-5-processing-n:127.0.0.1:59055_m%2Fz) [n:127.0.0.1:59055_m%2Fz ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:60163_m%2Fz for creating new replica
[junit4] 2> 1822045 INFO (qtp174427071-14955) [n:127.0.0.1:60163_m%2Fz ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n25&action=CREATE&collection=collection1&shard=shard1&wt=javabin&vers
[...truncated too long message...]
ect(Native Method)
[junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
[junit4] 2> Jan 19, 2018 6:23:10 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> SEVERE: There are still zombie threads that couldn't be terminated:
[junit4] 2> 1) Thread[id=16719, name=jetty-launcher-3081-thread-1-SendThread(127.0.0.1:46985), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=126, maxMBSortInHeap=7.8303602387354, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@4800dd8e), locale=sk, timezone=Etc/GMT-13
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=4,free=51534672,total=503316480
[junit4] 2> NOTE: All tests run in this JVM: [SolrSlf4jReporterTest, PropertiesRequestHandlerTest, TestExactSharedStatsCache, TestIndexingPerformance, TestInfoStreamLogging, MoveReplicaHDFSFailoverTest, SolrCoreCheckLockOnStartupTest, FileBasedSpellCheckerTest, TestObjectReleaseTracker, TestConfigSetsAPI, TestMiniSolrCloudClusterSSL, TestSolrQueryResponse, TestQuerySenderListener, TestFieldCollectionResource, TestBlobHandler, HttpSolrCallGetCoreTest, CdcrBidirectionalTest, RuleEngineTest, TestTlogReplica, SuggesterWFSTTest, CollectionStateFormat2Test, TestCloudSearcherWarming, CdcrRequestHandlerTest, TestPolicyCloud, EchoParamsTest, NotRequiredUniqueKeyTest, TestSerializedLuceneMatchVersion, ExitableDirectoryReaderTest, TestAuthorizationFramework, NumberUtilsTest, TestGenericDistributedQueue, SubstringBytesRefFilterTest, TestIntervalFaceting, LoggingHandlerTest, TestClusterStateMutator, TestWriterPerf, TestCloudManagedSchema, TestConfigSetsAPIZkFailure, CloneFieldUpdateProcessorFactoryTest, SolrGangliaReporterTest, SpatialHeatmapFacetsTest, TestHdfsCloudBackupRestore, TestRawTransformer, TestDistributedGrouping, TestCoreBackup, TestComponentsName, UUIDUpdateProcessorFallbackTest, CleanupOldIndexTest, MetricUtilsTest, TestRandomFaceting, TestPrepRecovery, BasicFunctionalityTest, HdfsWriteToMultipleCollectionsTest, BlockJoinFacetDistribTest, TestPullReplicaErrorHandling, TestValueSourceCache, TestUnifiedSolrHighlighter, TestHashPartitioner, AtomicUpdatesTest, TestConfigsApi, TestPushWriter, QueryParsingTest, TestExportWriter, TestNumericTokenStream, TestAddFieldRealTimeGet, TestSchemaManager, TestSQLHandlerNonCloud, TestLegacyNumericRangeQueryBuilder, TestSolrCoreParser, SliceStateTest, TestReversedWildcardFilterFactory, AddSchemaFieldsUpdateProcessorFactoryTest, DocValuesTest, TestBinaryResponseWriter, SyncSliceTest, ZkSolrClientTest, CSVRequestHandlerTest, DataDrivenBlockJoinTest, BasicAuthStandaloneTest, TestSchemaNameResource, CoreAdminRequestStatusTest, SimpleMLTQParserTest, SegmentsInfoRequestHandlerTest, TestAnalyzedSuggestions, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestDownShardTolerantSearch, HdfsRecoveryZkTest, ConfigSetsAPITest, OverriddenZkACLAndCredentialsProvidersTest, HdfsSyncSliceTest, RequestHandlersTest, SuggestComponentTest, TestSolrIndexConfig, AutoAddReplicasIntegrationTest, TestPHPSerializedResponseWriter, CollectionsAPIDistributedZkTest, NodeAddedTriggerTest, TestXmlQParserPlugin, TestCloudJSONFacetJoinDomain, TestDFRSimilarityFactory, SchemaApiFailureTest, DebugComponentTest, ClusterStateTest, AutoScalingHandlerTest, SimpleFacetsTest, TestLMJelinekMercerSimilarityFactory, TestPayloadCheckQParserPlugin, TestSolrConfigHandlerConcurrent, DeleteLastCustomShardedReplicaTest, TestRuleBasedAuthorizationPlugin, ClassificationUpdateProcessorIntegrationTest, HdfsNNFailoverTest, NoCacheHeaderTest, AliasIntegrationTest, JvmMetricsTest, BlobRepositoryCloudTest, UpdateRequestProcessorFactoryTest, CheckHdfsIndexTest, ZkControllerTest, TestDynamicFieldCollectionResource, HdfsChaosMonkeyNothingIsSafeTest, TestLRUCache, DeleteReplicaTest, TestAnalyzeInfixSuggestions, CachingDirectoryFactoryTest, PreAnalyzedFieldTest, ExecutePlanActionTest, TestStressInPlaceUpdates, TestJettySolrRunner, TimeRoutedAliasUpdateProcessorTest, TestSolrDeletionPolicy2, TestLuceneMatchVersion, TestLMDirichletSimilarityFactory, MoveReplicaTest, TestSubQueryTransformerDistrib, TestCorePropertiesReload, TestWordDelimiterFilterFactory, TestUseDocValuesAsStored, TestSmileRequest, FacetPivotSmallTest, IndexBasedSpellCheckerTest, TestFuzzyAnalyzedSuggestions, DistributedQueueTest, DefaultValueUpdateProcessorTest, TestFieldTypeCollectionResource, TestNoOpRegenerator, TestMinMaxOnMultiValuedField, DocValuesMultiTest, TestNumericRangeQuery32, TestConfig, TestSimpleTrackingShardHandler, TestSolrCloudWithHadoopAuthPlugin, DistributedExpandComponentTest, TestCustomSort, TestSolrCloudWithSecureImpersonation]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=694F82930B56CC0C -Dtests.slow=true -Dtests.locale=sk -Dtests.timezone=Etc/GMT-13 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J1 | TestSolrCloudWithSecureImpersonation (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
[junit4] > 1) Thread[id=16653, name=jetty-launcher-3081-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] > at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] > at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] > at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] > at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] > 2) Thread[id=16719, name=jetty-launcher-3081-thread-1-SendThread(127.0.0.1:46985), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > 3) Thread[id=16720, name=jetty-launcher-3081-thread-1-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
[junit4] > at __randomizedtesting.SeedInfo.seed([694F82930B56CC0C]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=16719, name=jetty-launcher-3081-thread-1-SendThread(127.0.0.1:46985), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > at __randomizedtesting.SeedInfo.seed([694F82930B56CC0C]:0)
[junit4] Completed [344/768 (2!)] on J1 in 64.23s, 12 tests, 2 errors <<< FAILURES!
[...truncated 49420 lines...]