You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/09/03 10:37:31 UTC
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 153 -
Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/153/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC
1 tests failed.
FAILED: org.apache.solr.cloud.LeaderElectionContextKeyTest.test
Error Message:
Stack Trace:
java.lang.AssertionError
at __randomizedtesting.SeedInfo.seed([2D96039CB4EB2C83:A5C23C461A17417B]:0)
at org.junit.Assert.fail(Assert.java:92)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertTrue(Assert.java:54)
at org.apache.solr.cloud.LeaderElectionContextKeyTest.test(LeaderElectionContextKeyTest.java:88)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11516 lines...]
[junit4] Suite: org.apache.solr.cloud.LeaderElectionContextKeyTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionContextKeyTest_2D96039CB4EB2C83-001/init-core-data-001
[junit4] 2> 704343 WARN (SUITE-LeaderElectionContextKeyTest-seed#[2D96039CB4EB2C83]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=56 numCloses=56
[junit4] 2> 704343 INFO (SUITE-LeaderElectionContextKeyTest-seed#[2D96039CB4EB2C83]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 704345 INFO (SUITE-LeaderElectionContextKeyTest-seed#[2D96039CB4EB2C83]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 704345 INFO (SUITE-LeaderElectionContextKeyTest-seed#[2D96039CB4EB2C83]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 704345 INFO (SUITE-LeaderElectionContextKeyTest-seed#[2D96039CB4EB2C83]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionContextKeyTest_2D96039CB4EB2C83-001/tempDir-001
[junit4] 2> 704345 INFO (SUITE-LeaderElectionContextKeyTest-seed#[2D96039CB4EB2C83]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 704345 INFO (Thread-3121) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 704345 INFO (Thread-3121) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 704347 ERROR (Thread-3121) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 704445 INFO (SUITE-LeaderElectionContextKeyTest-seed#[2D96039CB4EB2C83]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:64394
[junit4] 2> 704454 INFO (jetty-launcher-730-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 704456 INFO (jetty-launcher-730-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3f7c6fed{/solr,null,AVAILABLE}
[junit4] 2> 704572 INFO (jetty-launcher-730-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2303194b{HTTP/1.1,[http/1.1]}{127.0.0.1:54135}
[junit4] 2> 704572 INFO (jetty-launcher-730-thread-1) [ ] o.e.j.s.Server Started @707341ms
[junit4] 2> 704572 INFO (jetty-launcher-730-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54135}
[junit4] 2> 704572 ERROR (jetty-launcher-730-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 704572 INFO (jetty-launcher-730-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.1.0
[junit4] 2> 704572 INFO (jetty-launcher-730-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 704572 INFO (jetty-launcher-730-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 704572 INFO (jetty-launcher-730-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-03T09:25:30.837Z
[junit4] 2> 704636 INFO (jetty-launcher-730-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 704644 INFO (jetty-launcher-730-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64394/solr
[junit4] 2> 704783 INFO (jetty-launcher-730-thread-1) [n:127.0.0.1:54135_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 704784 INFO (jetty-launcher-730-thread-1) [n:127.0.0.1:54135_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:54135_solr
[junit4] 2> 704785 INFO (jetty-launcher-730-thread-1) [n:127.0.0.1:54135_solr ] o.a.s.c.Overseer Overseer (id=98594372361388035-127.0.0.1:54135_solr-n_0000000000) starting
[junit4] 2> 704798 INFO (jetty-launcher-730-thread-1) [n:127.0.0.1:54135_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54135_solr
[junit4] 2> 704803 INFO (zkCallback-737-thread-1-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 704995 INFO (jetty-launcher-730-thread-1) [n:127.0.0.1:54135_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54135.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f3a0633
[junit4] 2> 705011 INFO (jetty-launcher-730-thread-1) [n:127.0.0.1:54135_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54135.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f3a0633
[junit4] 2> 705011 INFO (jetty-launcher-730-thread-1) [n:127.0.0.1:54135_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54135.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f3a0633
[junit4] 2> 705012 INFO (jetty-launcher-730-thread-1) [n:127.0.0.1:54135_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionContextKeyTest_2D96039CB4EB2C83-001/tempDir-001/node1/.
[junit4] 2> 705030 INFO (SUITE-LeaderElectionContextKeyTest-seed#[2D96039CB4EB2C83]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 705031 INFO (SUITE-LeaderElectionContextKeyTest-seed#[2D96039CB4EB2C83]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64394/solr ready
[junit4] 2> 705040 INFO (qtp803726914-6570) [n:127.0.0.1:54135_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=config&maxShardsPerNode=1000&name=testCollection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 705043 INFO (OverseerThreadFactory-1834-thread-1-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.CreateCollectionCmd Create collection testCollection1
[junit4] 2> 705153 INFO (OverseerStateUpdate-98594372361388035-127.0.0.1:54135_solr-n_0000000000) [n:127.0.0.1:54135_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testCollection1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testCollection1_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:54135/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 705157 INFO (OverseerStateUpdate-98594372361388035-127.0.0.1:54135_solr-n_0000000000) [n:127.0.0.1:54135_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testCollection1",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"testCollection1_shard2_replica_n3",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:54135/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 705361 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=config&newCollection=true&name=testCollection1_shard1_replica_n1&action=CREATE&numShards=2&collection=testCollection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 705362 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 705364 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=testCollection1_shard2_replica_n3&action=CREATE&numShards=2&collection=testCollection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 705472 INFO (zkCallback-737-thread-1-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 705472 INFO (zkCallback-737-thread-2-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 706378 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 706378 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 706385 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.s.IndexSchema [testCollection1_shard2_replica_n3] Schema name=minimal
[junit4] 2> 706385 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.s.IndexSchema [testCollection1_shard1_replica_n1] Schema name=minimal
[junit4] 2> 706388 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 706388 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 706388 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'testCollection1_shard2_replica_n3' using configuration from collection testCollection1, trusted=true
[junit4] 2> 706388 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'testCollection1_shard1_replica_n1' using configuration from collection testCollection1, trusted=true
[junit4] 2> 706390 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54135.solr.core.testCollection1.shard1.replica_n1' (registry 'solr.core.testCollection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f3a0633
[junit4] 2> 706390 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 706390 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SolrCore [[testCollection1_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionContextKeyTest_2D96039CB4EB2C83-001/tempDir-001/node1/testCollection1_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionContextKeyTest_2D96039CB4EB2C83-001/tempDir-001/node1/./testCollection1_shard1_replica_n1/data/]
[junit4] 2> 706394 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54135.solr.core.testCollection1.shard2.replica_n3' (registry 'solr.core.testCollection1.shard2.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f3a0633
[junit4] 2> 706394 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 706394 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SolrCore [[testCollection1_shard2_replica_n3] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionContextKeyTest_2D96039CB4EB2C83-001/tempDir-001/node1/testCollection1_shard2_replica_n3], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionContextKeyTest_2D96039CB4EB2C83-001/tempDir-001/node1/./testCollection1_shard2_replica_n3/data/]
[junit4] 2> 706467 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 706467 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 706467 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 706467 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 706468 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 706468 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 706468 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 706468 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 706470 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@589eedfc[testCollection1_shard1_replica_n1] main]
[junit4] 2> 706470 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@7bc91777[testCollection1_shard2_replica_n3] main]
[junit4] 2> 706471 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
[junit4] 2> 706471 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
[junit4] 2> 706472 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
[junit4] 2> 706472 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
[junit4] 2> 706472 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 706472 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 706473 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577509960011481088
[junit4] 2> 706473 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577509960011481088
[junit4] 2> 706473 INFO (searcherExecutor-1837-thread-1-processing-n:127.0.0.1:54135_solr x:testCollection1_shard2_replica_n3 s:shard2 c:testCollection1 r:core_node4) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SolrCore [testCollection1_shard2_replica_n3] Registered new searcher Searcher@7bc91777[testCollection1_shard2_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 706478 INFO (searcherExecutor-1838-thread-1-processing-n:127.0.0.1:54135_solr x:testCollection1_shard1_replica_n1 s:shard1 c:testCollection1 r:core_node2) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SolrCore [testCollection1_shard1_replica_n1] Registered new searcher Searcher@589eedfc[testCollection1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 706485 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 706485 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 706485 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:54135/solr/testCollection1_shard2_replica_n3/
[junit4] 2> 706485 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 706485 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SyncStrategy http://127.0.0.1:54135/solr/testCollection1_shard2_replica_n3/ has no replicas
[junit4] 2> 706485 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 706485 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 706485 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 706486 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:54135/solr/testCollection1_shard1_replica_n1/
[junit4] 2> 706486 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 706486 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:54135/solr/testCollection1_shard1_replica_n1/ has no replicas
[junit4] 2> 706486 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 706497 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:54135/solr/testCollection1_shard1_replica_n1/ shard1
[junit4] 2> 706498 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:54135/solr/testCollection1_shard2_replica_n3/ shard2
[junit4] 2> 706605 INFO (zkCallback-737-thread-2-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 706605 INFO (zkCallback-737-thread-1-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 706650 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 706650 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 706652 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=testCollection1_shard2_replica_n3&action=CREATE&numShards=2&collection=testCollection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1288
[junit4] 2> 706657 INFO (qtp803726914-6572) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=config&newCollection=true&name=testCollection1_shard1_replica_n1&action=CREATE&numShards=2&collection=testCollection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1297
[junit4] 2> 706661 INFO (qtp803726914-6570) [n:127.0.0.1:54135_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 706761 INFO (zkCallback-737-thread-2-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 706762 INFO (zkCallback-737-thread-1-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 707045 INFO (OverseerCollectionConfigSetProcessor-98594372361388035-127.0.0.1:54135_solr-n_0000000000) [n:127.0.0.1:54135_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 707662 INFO (qtp803726914-6570) [n:127.0.0.1:54135_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=config&maxShardsPerNode=1000&name=testCollection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=2622
[junit4] 2> 707663 INFO (qtp803726914-6573) [n:127.0.0.1:54135_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=config&maxShardsPerNode=1000&name=testCollection2&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 707665 INFO (OverseerThreadFactory-1834-thread-2-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.CreateCollectionCmd Create collection testCollection2
[junit4] 2> 707876 INFO (OverseerStateUpdate-98594372361388035-127.0.0.1:54135_solr-n_0000000000) [n:127.0.0.1:54135_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testCollection2",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testCollection2_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:54135/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 707879 INFO (OverseerStateUpdate-98594372361388035-127.0.0.1:54135_solr-n_0000000000) [n:127.0.0.1:54135_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testCollection2",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"testCollection2_shard2_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:54135/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 708077 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=testCollection2_shard2_replica_n2&action=CREATE&numShards=2&collection=testCollection2&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 708077 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=testCollection2_shard1_replica_n1&action=CREATE&numShards=2&collection=testCollection2&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 708185 INFO (zkCallback-737-thread-1-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
[junit4] 2> 708185 INFO (zkCallback-737-thread-2-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
[junit4] 2> 709089 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 709094 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.s.IndexSchema [testCollection2_shard1_replica_n1] Schema name=minimal
[junit4] 2> 709096 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 709096 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'testCollection2_shard1_replica_n1' using configuration from collection testCollection2, trusted=true
[junit4] 2> 709097 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54135.solr.core.testCollection2.shard1.replica_n1' (registry 'solr.core.testCollection2.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f3a0633
[junit4] 2> 709097 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 709097 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SolrCore [[testCollection2_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionContextKeyTest_2D96039CB4EB2C83-001/tempDir-001/node1/testCollection2_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionContextKeyTest_2D96039CB4EB2C83-001/tempDir-001/node1/./testCollection2_shard1_replica_n1/data/]
[junit4] 2> 709123 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 709146 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.s.IndexSchema [testCollection2_shard2_replica_n2] Schema name=minimal
[junit4] 2> 709148 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 709148 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'testCollection2_shard2_replica_n2' using configuration from collection testCollection2, trusted=true
[junit4] 2> 709148 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54135.solr.core.testCollection2.shard2.replica_n2' (registry 'solr.core.testCollection2.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f3a0633
[junit4] 2> 709148 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 709148 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SolrCore [[testCollection2_shard2_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionContextKeyTest_2D96039CB4EB2C83-001/tempDir-001/node1/testCollection2_shard2_replica_n2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionContextKeyTest_2D96039CB4EB2C83-001/tempDir-001/node1/./testCollection2_shard2_replica_n2/data/]
[junit4] 2> 709197 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 709197 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 709198 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 709198 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 709198 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@1dddc81e[testCollection2_shard2_replica_n2] main]
[junit4] 2> 709213 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
[junit4] 2> 709214 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
[junit4] 2> 709214 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 709214 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577509962885627904
[junit4] 2> 709216 INFO (searcherExecutor-1848-thread-1-processing-n:127.0.0.1:54135_solr x:testCollection2_shard2_replica_n2 s:shard2 c:testCollection2 r:core_node4) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SolrCore [testCollection2_shard2_replica_n2] Registered new searcher Searcher@1dddc81e[testCollection2_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 709221 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 709221 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 709221 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:54135/solr/testCollection2_shard2_replica_n2/
[junit4] 2> 709221 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 709221 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SyncStrategy http://127.0.0.1:54135/solr/testCollection2_shard2_replica_n2/ has no replicas
[junit4] 2> 709222 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 709229 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 709229 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 709230 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 709230 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 709231 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1e543f40[testCollection2_shard1_replica_n1] main]
[junit4] 2> 709233 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
[junit4] 2> 709234 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
[junit4] 2> 709235 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 709235 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577509962907648000
[junit4] 2> 709244 INFO (searcherExecutor-1847-thread-1-processing-n:127.0.0.1:54135_solr x:testCollection2_shard1_replica_n1 s:shard1 c:testCollection2 r:core_node3) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SolrCore [testCollection2_shard1_replica_n1] Registered new searcher Searcher@1e543f40[testCollection2_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 709248 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:54135/solr/testCollection2_shard2_replica_n2/ shard2
[junit4] 2> 709250 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 709250 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 709250 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:54135/solr/testCollection2_shard1_replica_n1/
[junit4] 2> 709250 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 709250 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:54135/solr/testCollection2_shard1_replica_n1/ has no replicas
[junit4] 2> 709250 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 709260 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:54135/solr/testCollection2_shard1_replica_n1/ shard1
[junit4] 2> 709365 INFO (zkCallback-737-thread-2-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
[junit4] 2> 709365 INFO (zkCallback-737-thread-1-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
[junit4] 2> 709400 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 709402 INFO (qtp803726914-6567) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=testCollection2_shard2_replica_n2&action=CREATE&numShards=2&collection=testCollection2&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1324
[junit4] 2> 709411 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 709414 INFO (qtp803726914-6574) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=testCollection2_shard1_replica_n1&action=CREATE&numShards=2&collection=testCollection2&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1337
[junit4] 2> 709418 INFO (qtp803726914-6573) [n:127.0.0.1:54135_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 709520 INFO (zkCallback-737-thread-2-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
[junit4] 2> 709520 INFO (zkCallback-737-thread-1-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
[junit4] 2> 709666 INFO (OverseerCollectionConfigSetProcessor-98594372361388035-127.0.0.1:54135_solr-n_0000000000) [n:127.0.0.1:54135_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 710430 INFO (qtp803726914-6573) [n:127.0.0.1:54135_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=config&maxShardsPerNode=1000&name=testCollection2&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=2767
[junit4] 2> 710430 INFO (SUITE-LeaderElectionContextKeyTest-seed#[2D96039CB4EB2C83]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: testCollection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 710431 INFO (SUITE-LeaderElectionContextKeyTest-seed#[2D96039CB4EB2C83]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: testCollection1
[junit4] 2> 710431 INFO (SUITE-LeaderElectionContextKeyTest-seed#[2D96039CB4EB2C83]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: testCollection2 failOnTimeout:true timeout (sec):30
[junit4] 2> 710431 INFO (SUITE-LeaderElectionContextKeyTest-seed#[2D96039CB4EB2C83]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: testCollection2
[junit4] 2> 710443 INFO (TEST-LeaderElectionContextKeyTest.test-seed#[2D96039CB4EB2C83]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 710452 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testCollection1.shard2.replica_n3, tag=null
[junit4] 2> 710452 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@152d837f: rootName = solr_54135, domain = solr.core.testCollection1.shard2.replica_n3, service url = null, agent id = null] for registry solr.core.testCollection1.shard2.replica_n3 / com.codahale.metrics.MetricRegistry@72957b67
[junit4] 2> 710469 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr ] o.a.s.c.SolrCore [testCollection1_shard2_replica_n3] CLOSING SolrCore org.apache.solr.core.SolrCore@73a882c6
[junit4] 2> 710469 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testCollection1.shard2.replica_n3, tag=1940423366
[junit4] 2> 710469 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testCollection1.shard2.leader, tag=1940423366
[junit4] 2> 710472 INFO (qtp803726914-6569) [n:127.0.0.1:54135_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=false&core=testCollection1_shard2_replica_n3&deleteDataDir=false&action=UNLOAD&indexInfo=true&wt=javabin&version=2&deleteIndex=false} status=0 QTime=19
[junit4] 2> 710478 INFO (TEST-LeaderElectionContextKeyTest.test-seed#[2D96039CB4EB2C83]) [ ] o.a.s.SolrTestCaseJ4 ###Ending test
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=LeaderElectionContextKeyTest -Dtests.method=test -Dtests.seed=2D96039CB4EB2C83 -Dtests.slow=true -Dtests.locale=id -Dtests.timezone=Asia/Tehran -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] FAILURE 0.05s J0 | LeaderElectionContextKeyTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError
[junit4] > at __randomizedtesting.SeedInfo.seed([2D96039CB4EB2C83:A5C23C461A17417B]:0)
[junit4] > at org.apache.solr.cloud.LeaderElectionContextKeyTest.test(LeaderElectionContextKeyTest.java:88)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 710486 INFO (jetty-closer-731-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@2303194b{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 710487 INFO (jetty-closer-731-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2082151472
[junit4] 2> 710487 INFO (jetty-closer-731-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 710487 INFO (jetty-closer-731-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6c5aeb2a: rootName = solr_54135, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@2902971a
[junit4] 2> 710492 INFO (jetty-closer-731-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 710492 INFO (jetty-closer-731-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@71e92925: rootName = solr_54135, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@591b5320
[junit4] 2> 710496 INFO (jetty-closer-731-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 710496 INFO (jetty-closer-731-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3f753f6: rootName = solr_54135, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@4da0f1a4
[junit4] 2> 710498 INFO (jetty-closer-731-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 710499 INFO (coreCloseExecutor-1857-thread-2) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SolrCore [testCollection2_shard2_replica_n2] CLOSING SolrCore org.apache.solr.core.SolrCore@558e45df
[junit4] 2> 710499 INFO (coreCloseExecutor-1857-thread-2) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testCollection2.shard2.replica_n2, tag=1435387359
[junit4] 2> 710499 INFO (coreCloseExecutor-1857-thread-1) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SolrCore [testCollection1_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@3943d816
[junit4] 2> 710499 INFO (coreCloseExecutor-1857-thread-2) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@75ab4899: rootName = solr_54135, domain = solr.core.testCollection2.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.testCollection2.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@6b1128f6
[junit4] 2> 710502 INFO (coreCloseExecutor-1857-thread-3) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SolrCore [testCollection2_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@13d85a30
[junit4] 2> 710510 INFO (coreCloseExecutor-1857-thread-2) [n:127.0.0.1:54135_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testCollection2.shard2.leader, tag=1435387359
[junit4] 2> 710513 INFO (coreCloseExecutor-1857-thread-1) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testCollection1.shard1.replica_n1, tag=960747542
[junit4] 2> 710513 INFO (coreCloseExecutor-1857-thread-1) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6d9f5e42: rootName = solr_54135, domain = solr.core.testCollection1.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.testCollection1.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@3deb7a98
[junit4] 2> 710521 INFO (coreCloseExecutor-1857-thread-1) [n:127.0.0.1:54135_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testCollection1.shard1.leader, tag=960747542
[junit4] 2> 710521 INFO (coreCloseExecutor-1857-thread-3) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testCollection2.shard1.replica_n1, tag=332945968
[junit4] 2> 710522 INFO (coreCloseExecutor-1857-thread-3) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@643e78d0: rootName = solr_54135, domain = solr.core.testCollection2.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.testCollection2.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@22d5917d
[junit4] 2> 710539 INFO (coreCloseExecutor-1857-thread-3) [n:127.0.0.1:54135_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testCollection2.shard1.leader, tag=332945968
[junit4] 2> 710545 INFO (jetty-closer-731-thread-1) [ ] o.a.s.c.Overseer Overseer (id=98594372361388035-127.0.0.1:54135_solr-n_0000000000) closing
[junit4] 2> 710546 INFO (OverseerStateUpdate-98594372361388035-127.0.0.1:54135_solr-n_0000000000) [n:127.0.0.1:54135_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:54135_solr
[junit4] 2> 710550 WARN (zkCallback-737-thread-2-processing-n:127.0.0.1:54135_solr) [n:127.0.0.1:54135_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 710551 INFO (jetty-closer-731-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3f7c6fed{/solr,null,UNAVAILABLE}
[junit4] 2> 710553 ERROR (SUITE-LeaderElectionContextKeyTest-seed#[2D96039CB4EB2C83]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 710556 INFO (SUITE-LeaderElectionContextKeyTest-seed#[2D96039CB4EB2C83]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:64394 64394
[junit4] 2> 721373 INFO (Thread-3121) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:64394 64394
[junit4] 2> 721374 WARN (Thread-3121) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/aliases.json
[junit4] 2> 2 /solr/clusterprops.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/collections/testCollection2/state.json
[junit4] 2> 5 /solr/collections/testCollection1/state.json
[junit4] 2> 2 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/live_nodes
[junit4] 2> 2 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionContextKeyTest_2D96039CB4EB2C83-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=293, maxMBSortInHeap=5.010468908497767, sim=RandomSimilarity(queryNorm=true): {}, locale=id, timezone=Asia/Tehran
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=160865152,total=518979584
[junit4] 2> NOTE: All tests run in this JVM: [BJQParserTest, TestStressRecovery, SSLMigrationTest, FastVectorHighlighterTest, JavabinLoaderTest, BinaryUpdateRequestHandlerTest, HdfsRestartWhileUpdatingTest, TestXIncludeConfig, TestPayloadCheckQParserPlugin, ClassificationUpdateProcessorTest, TestPullReplicaErrorHandling, ExplicitHLLTest, TestRestoreCore, TestPointFields, TestAtomicUpdateErrorCases, CloudMLTQParserTest, TestComplexPhraseLeadingWildcard, ManagedSchemaRoundRobinCloudTest, CdcrReplicationHandlerTest, MultiThreadedOCPTest, BooleanFieldTest, HdfsCollectionsAPIDistributedZkTest, TestBackupRepositoryFactory, TestStressReorder, DeleteNodeTest, TestDeleteCollectionOnDownNodes, TestFoldingMultitermQuery, ScriptEngineTest, SpatialFilterTest, HdfsChaosMonkeyNothingIsSafeTest, TestHdfsUpdateLog, AssignBackwardCompatibilityTest, TestFieldCacheSort, CircularListTest, TestQuerySenderNoQuery, TestReplicationHandlerBackup, TestConfigSetsAPI, TestRecoveryHdfs, TestCustomStream, TestRandomRequestDistribution, PKIAuthenticationIntegrationTest, ResponseHeaderTest, TestFieldCache, TestPivotHelperCode, TestSolrConfigHandlerConcurrent, GraphQueryTest, LeaderElectionContextKeyTest]
[junit4] Completed [118/731 (1!)] on J0 in 17.05s, 1 test, 1 failure <<< FAILURES!
[...truncated 48234 lines...]
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 155 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/155/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest
Error Message:
Timeout waiting for all live and active
Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
at __randomizedtesting.SeedInfo.seed([B940B2629F487985:3A36ED9049317724]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest(TestCloudRecovery.java:184)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12057 lines...]
[junit4] Suite: org.apache.solr.cloud.TestCloudRecovery
[junit4] 2> 1292016 INFO (SUITE-TestCloudRecovery-seed#[B940B2629F487985]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_B940B2629F487985-001/init-core-data-001
[junit4] 2> 1292017 INFO (SUITE-TestCloudRecovery-seed#[B940B2629F487985]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 1292018 INFO (SUITE-TestCloudRecovery-seed#[B940B2629F487985]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 1292019 INFO (SUITE-TestCloudRecovery-seed#[B940B2629F487985]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_B940B2629F487985-001/tempDir-001
[junit4] 2> 1292019 INFO (SUITE-TestCloudRecovery-seed#[B940B2629F487985]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1292019 INFO (Thread-2428) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1292019 INFO (Thread-2428) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1292020 ERROR (Thread-2428) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1292119 INFO (SUITE-TestCloudRecovery-seed#[B940B2629F487985]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:51205
[junit4] 2> 1292128 INFO (jetty-launcher-2559-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1292128 INFO (jetty-launcher-2559-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1292130 INFO (jetty-launcher-2559-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@42f4a2a0{/solr,null,AVAILABLE}
[junit4] 2> 1292130 INFO (jetty-launcher-2559-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7cce0fa6{/solr,null,AVAILABLE}
[junit4] 2> 1292130 INFO (jetty-launcher-2559-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@561c4fd9{SSL,[ssl, http/1.1]}{127.0.0.1:41175}
[junit4] 2> 1292130 INFO (jetty-launcher-2559-thread-2) [ ] o.e.j.s.Server Started @1294149ms
[junit4] 2> 1292130 INFO (jetty-launcher-2559-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41175}
[junit4] 2> 1292130 INFO (jetty-launcher-2559-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6035efa0{SSL,[ssl, http/1.1]}{127.0.0.1:47999}
[junit4] 2> 1292130 INFO (jetty-launcher-2559-thread-1) [ ] o.e.j.s.Server Started @1294149ms
[junit4] 2> 1292130 INFO (jetty-launcher-2559-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=47999}
[junit4] 2> 1292130 ERROR (jetty-launcher-2559-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1292130 ERROR (jetty-launcher-2559-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1292130 INFO (jetty-launcher-2559-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.1.0
[junit4] 2> 1292130 INFO (jetty-launcher-2559-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1292130 INFO (jetty-launcher-2559-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.1.0
[junit4] 2> 1292130 INFO (jetty-launcher-2559-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1292130 INFO (jetty-launcher-2559-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1292130 INFO (jetty-launcher-2559-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-04T08:50:45.009Z
[junit4] 2> 1292130 INFO (jetty-launcher-2559-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1292131 INFO (jetty-launcher-2559-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-04T08:50:45.010Z
[junit4] 2> 1292140 INFO (jetty-launcher-2559-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1292141 INFO (jetty-launcher-2559-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1292146 INFO (jetty-launcher-2559-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51205/solr
[junit4] 2> 1292147 INFO (jetty-launcher-2559-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51205/solr
[junit4] 2> 1292155 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e4c15b6230004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1292185 INFO (jetty-launcher-2559-thread-2) [n:127.0.0.1:41175_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1292186 INFO (jetty-launcher-2559-thread-2) [n:127.0.0.1:41175_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:41175_solr
[junit4] 2> 1292187 INFO (jetty-launcher-2559-thread-2) [n:127.0.0.1:41175_solr ] o.a.s.c.Overseer Overseer (id=98599897982500870-127.0.0.1:41175_solr-n_0000000000) starting
[junit4] 2> 1292195 INFO (jetty-launcher-2559-thread-2) [n:127.0.0.1:41175_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41175_solr
[junit4] 2> 1292196 INFO (OverseerStateUpdate-98599897982500870-127.0.0.1:41175_solr-n_0000000000) [n:127.0.0.1:41175_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1292275 INFO (jetty-launcher-2559-thread-1) [n:127.0.0.1:47999_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1292277 INFO (jetty-launcher-2559-thread-1) [n:127.0.0.1:47999_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1292279 INFO (jetty-launcher-2559-thread-1) [n:127.0.0.1:47999_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47999_solr
[junit4] 2> 1292281 INFO (zkCallback-2571-thread-1-processing-n:127.0.0.1:41175_solr) [n:127.0.0.1:41175_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1292282 INFO (zkCallback-2570-thread-1-processing-n:127.0.0.1:47999_solr) [n:127.0.0.1:47999_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1292346 INFO (jetty-launcher-2559-thread-1) [n:127.0.0.1:47999_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47999.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a9a1ef1
[junit4] 2> 1292352 INFO (jetty-launcher-2559-thread-1) [n:127.0.0.1:47999_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47999.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a9a1ef1
[junit4] 2> 1292352 INFO (jetty-launcher-2559-thread-1) [n:127.0.0.1:47999_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47999.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a9a1ef1
[junit4] 2> 1292353 INFO (jetty-launcher-2559-thread-1) [n:127.0.0.1:47999_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_B940B2629F487985-001/tempDir-001/node1/.
[junit4] 2> 1292362 INFO (jetty-launcher-2559-thread-2) [n:127.0.0.1:41175_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41175.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a9a1ef1
[junit4] 2> 1292367 INFO (jetty-launcher-2559-thread-2) [n:127.0.0.1:41175_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41175.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a9a1ef1
[junit4] 2> 1292368 INFO (jetty-launcher-2559-thread-2) [n:127.0.0.1:41175_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41175.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a9a1ef1
[junit4] 2> 1292368 INFO (jetty-launcher-2559-thread-2) [n:127.0.0.1:41175_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_B940B2629F487985-001/tempDir-001/node2/.
[junit4] 2> 1292389 INFO (SUITE-TestCloudRecovery-seed#[B940B2629F487985]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1292391 INFO (SUITE-TestCloudRecovery-seed#[B940B2629F487985]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:51205/solr ready
[junit4] 2> 1292407 INFO (qtp1538454288-12175) [n:127.0.0.1:41175_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=0&collection.configName=config&maxShardsPerNode=2&name=collection1&nrtReplicas=0&action=CREATE&numShards=2&tlogReplicas=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1292410 INFO (OverseerThreadFactory-5201-thread-1-processing-n:127.0.0.1:41175_solr) [n:127.0.0.1:41175_solr ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1292522 INFO (OverseerStateUpdate-98599897982500870-127.0.0.1:41175_solr-n_0000000000) [n:127.0.0.1:41175_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"collection1_shard1_replica_t1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:41175/solr",
[junit4] 2> "type":"TLOG"}
[junit4] 2> 1292526 INFO (OverseerStateUpdate-98599897982500870-127.0.0.1:41175_solr-n_0000000000) [n:127.0.0.1:41175_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"collection1_shard1_replica_t2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:47999/solr",
[junit4] 2> "type":"TLOG"}
[junit4] 2> 1292529 INFO (OverseerStateUpdate-98599897982500870-127.0.0.1:41175_solr-n_0000000000) [n:127.0.0.1:41175_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"collection1_shard2_replica_t4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:41175/solr",
[junit4] 2> "type":"TLOG"}
[junit4] 2> 1292533 INFO (OverseerStateUpdate-98599897982500870-127.0.0.1:41175_solr-n_0000000000) [n:127.0.0.1:41175_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"collection1_shard2_replica_t6",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:47999/solr",
[junit4] 2> "type":"TLOG"}
[junit4] 2> 1292739 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 1292739 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1292739 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t4&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 1292761 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t2&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 1292761 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t6&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 1292762 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1292877 INFO (zkCallback-2571-thread-1-processing-n:127.0.0.1:41175_solr) [n:127.0.0.1:41175_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1292877 INFO (zkCallback-2570-thread-1-processing-n:127.0.0.1:47999_solr) [n:127.0.0.1:47999_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1292877 INFO (zkCallback-2571-thread-2-processing-n:127.0.0.1:41175_solr) [n:127.0.0.1:41175_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1292877 INFO (zkCallback-2570-thread-2-processing-n:127.0.0.1:47999_solr) [n:127.0.0.1:47999_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1293787 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 1293787 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 1293790 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 1293793 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.IndexSchema [collection1_shard1_replica_t1] Schema name=minimal
[junit4] 2> 1293795 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1293795 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t1' using configuration from collection collection1, trusted=true
[junit4] 2> 1293795 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41175.solr.core.collection1.shard1.replica_t1' (registry 'solr.core.collection1.shard1.replica_t1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a9a1ef1
[junit4] 2> 1293795 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1293795 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore [[collection1_shard1_replica_t1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_B940B2629F487985-001/tempDir-001/node2/collection1_shard1_replica_t1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_B940B2629F487985-001/tempDir-001/node2/./collection1_shard1_replica_t1/data/]
[junit4] 2> 1293796 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.s.IndexSchema [collection1_shard2_replica_t4] Schema name=minimal
[junit4] 2> 1293798 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1293798 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t4' using configuration from collection collection1, trusted=true
[junit4] 2> 1293798 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41175.solr.core.collection1.shard2.replica_t4' (registry 'solr.core.collection1.shard2.replica_t4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a9a1ef1
[junit4] 2> 1293798 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1293798 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SolrCore [[collection1_shard2_replica_t4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_B940B2629F487985-001/tempDir-001/node2/collection1_shard2_replica_t4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_B940B2629F487985-001/tempDir-001/node2/./collection1_shard2_replica_t4/data/]
[junit4] 2> 1293800 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 1294314 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.s.IndexSchema [collection1_shard1_replica_t2] Schema name=minimal
[junit4] 2> 1294317 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1294317 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t2' using configuration from collection collection1, trusted=true
[junit4] 2> 1294318 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47999.solr.core.collection1.shard1.replica_t2' (registry 'solr.core.collection1.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a9a1ef1
[junit4] 2> 1294318 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1294318 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SolrCore [[collection1_shard1_replica_t2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_B940B2629F487985-001/tempDir-001/node1/collection1_shard1_replica_t2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_B940B2629F487985-001/tempDir-001/node1/./collection1_shard1_replica_t2/data/]
[junit4] 2> 1294326 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.IndexSchema [collection1_shard2_replica_t6] Schema name=minimal
[junit4] 2> 1294327 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1294327 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t6' using configuration from collection collection1, trusted=true
[junit4] 2> 1294327 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47999.solr.core.collection1.shard2.replica_t6' (registry 'solr.core.collection1.shard2.replica_t6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a9a1ef1
[junit4] 2> 1294327 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1294327 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore [[collection1_shard2_replica_t6] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_B940B2629F487985-001/tempDir-001/node1/collection1_shard2_replica_t6], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_B940B2629F487985-001/tempDir-001/node1/./collection1_shard2_replica_t6/data/]
[junit4] 2> 1294356 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1294356 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1294370 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1294370 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1294371 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1294371 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1294372 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.s.SolrIndexSearcher Opening [Searcher@734fd938[collection1_shard2_replica_t4] main]
[junit4] 2> 1294373 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
[junit4] 2> 1294374 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
[junit4] 2> 1294374 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1294374 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577598370188361728
[junit4] 2> 1294376 INFO (searcherExecutor-5207-thread-1-processing-n:127.0.0.1:41175_solr x:collection1_shard2_replica_t4 s:shard2 c:collection1 r:core_node7) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SolrCore [collection1_shard2_replica_t4] Registered new searcher Searcher@734fd938[collection1_shard2_replica_t4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1294378 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1294378 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1294379 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1294379 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1294380 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@7a538087[collection1_shard1_replica_t2] main]
[junit4] 2> 1294383 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
[junit4] 2> 1294383 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
[junit4] 2> 1294384 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1294384 INFO (searcherExecutor-5208-thread-1-processing-n:127.0.0.1:47999_solr x:collection1_shard1_replica_t2 s:shard1 c:collection1 r:core_node5) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SolrCore [collection1_shard1_replica_t2] Registered new searcher Searcher@7a538087[collection1_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1294384 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1294384 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577598370198847488
[junit4] 2> 1294384 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1294385 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1294385 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1294387 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.SolrIndexSearcher Opening [Searcher@499fa13a[collection1_shard2_replica_t6] main]
[junit4] 2> 1294387 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1294387 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1294388 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=9999ms
[junit4] 2> 1294389 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@56dac31b[collection1_shard1_replica_t1] main]
[junit4] 2> 1294390 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
[junit4] 2> 1294390 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
[junit4] 2> 1294391 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1294391 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577598370206187520
[junit4] 2> 1294391 INFO (searcherExecutor-5209-thread-1-processing-n:127.0.0.1:47999_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore [collection1_shard2_replica_t6] Registered new searcher Searcher@499fa13a[collection1_shard2_replica_t6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1294392 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
[junit4] 2> 1294392 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
[junit4] 2> 1294393 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1294395 INFO (searcherExecutor-5206-thread-1-processing-n:127.0.0.1:41175_solr x:collection1_shard1_replica_t1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore [collection1_shard1_replica_t1] Registered new searcher Searcher@56dac31b[collection1_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1294396 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 1294396 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577598370211430400
[junit4] 2> 1294503 INFO (zkCallback-2571-thread-2-processing-n:127.0.0.1:41175_solr) [n:127.0.0.1:41175_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1294504 INFO (zkCallback-2570-thread-1-processing-n:127.0.0.1:47999_solr) [n:127.0.0.1:47999_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1294505 INFO (zkCallback-2571-thread-1-processing-n:127.0.0.1:41175_solr) [n:127.0.0.1:41175_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1294505 INFO (zkCallback-2570-thread-2-processing-n:127.0.0.1:47999_solr) [n:127.0.0.1:47999_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1294890 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1294890 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1294890 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:41175/solr/collection1_shard2_replica_t4/
[junit4] 2> 1294890 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.PeerSync PeerSync: core=collection1_shard2_replica_t4 url=https://127.0.0.1:41175/solr START replicas=[https://127.0.0.1:47999/solr/collection1_shard2_replica_t6/] nUpdates=100
[junit4] 2> 1294895 INFO (qtp51216966-12183) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.S.Request [collection1_shard2_replica_t6] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1294898 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1294898 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1294898 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:47999/solr/collection1_shard1_replica_t2/
[junit4] 2> 1294898 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_t2 url=https://127.0.0.1:47999/solr START replicas=[https://127.0.0.1:41175/solr/collection1_shard1_replica_t1/] nUpdates=100
[junit4] 2> 1294902 INFO (qtp1538454288-12170) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.S.Request [collection1_shard1_replica_t1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1295191 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.PeerSync PeerSync: core=collection1_shard2_replica_t4 url=https://127.0.0.1:41175/solr DONE. We have no versions. sync failed.
[junit4] 2> 1295191 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 1295191 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 1295191 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1295192 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ZkController collection1_shard2_replica_t4 stopping background replication from leader
[junit4] 2> 1295197 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:41175/solr/collection1_shard2_replica_t4/ shard2
[junit4] 2> 1295200 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_t2 url=https://127.0.0.1:47999/solr DONE. We have no versions. sync failed.
[junit4] 2> 1295200 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 1295200 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 1295200 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1295200 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ZkController collection1_shard1_replica_t2 stopping background replication from leader
[junit4] 2> 1295205 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:47999/solr/collection1_shard1_replica_t2/ shard1
[junit4] 2> 1295309 INFO (zkCallback-2571-thread-1-processing-n:127.0.0.1:41175_solr) [n:127.0.0.1:41175_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1295309 INFO (zkCallback-2570-thread-1-processing-n:127.0.0.1:47999_solr) [n:127.0.0.1:47999_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1295309 INFO (zkCallback-2570-thread-2-processing-n:127.0.0.1:47999_solr) [n:127.0.0.1:47999_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1295309 INFO (zkCallback-2571-thread-2-processing-n:127.0.0.1:41175_solr) [n:127.0.0.1:41175_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1295349 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1295350 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node7&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t4&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2613
[junit4] 2> 1295356 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1295358 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t2&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2597
[junit4] 2> 1295400 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.ZkController collection1_shard2_replica_t6 starting background replication from leader
[junit4] 2> 1295400 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 1295401 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 1295402 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Replica core_node7 is leader but it's state is down, skipping replication
[junit4] 2> 1295403 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node8&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t6&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2642
[junit4] 2> 1295405 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.ZkController collection1_shard1_replica_t1 starting background replication from leader
[junit4] 2> 1295405 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 1295406 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 1295406 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Replica core_node5 is leader but it's state is down, skipping replication
[junit4] 2> 1295410 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2671
[junit4] 2> 1295414 INFO (qtp1538454288-12175) [n:127.0.0.1:41175_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1295515 INFO (zkCallback-2570-thread-2-processing-n:127.0.0.1:47999_solr) [n:127.0.0.1:47999_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1295515 INFO (zkCallback-2571-thread-2-processing-n:127.0.0.1:41175_solr) [n:127.0.0.1:41175_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1295515 INFO (zkCallback-2571-thread-1-processing-n:127.0.0.1:41175_solr) [n:127.0.0.1:41175_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1295516 INFO (zkCallback-2570-thread-1-processing-n:127.0.0.1:47999_solr) [n:127.0.0.1:47999_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1296415 INFO (OverseerCollectionConfigSetProcessor-98599897982500870-127.0.0.1:41175_solr-n_0000000000) [n:127.0.0.1:41175_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1296415 INFO (qtp1538454288-12175) [n:127.0.0.1:41175_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=0&collection.configName=config&maxShardsPerNode=2&name=collection1&nrtReplicas=0&action=CREATE&numShards=2&tlogReplicas=2&wt=javabin&version=2} status=0 QTime=4009
[junit4] 2> 1296416 INFO (SUITE-TestCloudRecovery-seed#[B940B2629F487985]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 1296416 INFO (SUITE-TestCloudRecovery-seed#[B940B2629F487985]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 1296430 INFO (TEST-TestCloudRecovery.leaderRecoverFromLogOnStartupTest-seed#[B940B2629F487985]) [ ] o.a.s.SolrTestCaseJ4 ###Starting leaderRecoverFromLogOnStartupTest
[junit4] 2> 1296443 INFO (qtp1538454288-12178) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t1] webapp=/solr path=/update params={update.distrib=FROMLEADER&_version_=-1577598372352622592&distrib.from=https://127.0.0.1:47999/solr/collection1_shard1_replica_t2/&wt=javabin&version=2}{deleteByQuery=*:* (-1577598372352622592)} 0 0
[junit4] 2> 1296446 INFO (qtp51216966-12182) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=https://127.0.0.1:41175/solr/collection1_shard2_replica_t4/&wt=javabin&version=2}{deleteByQuery=*:* (-1577598372352622592)} 0 7
[junit4] 2> 1296448 INFO (qtp51216966-12184) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t6] webapp=/solr path=/update params={update.distrib=FROMLEADER&_version_=-1577598372346331136&distrib.from=https://127.0.0.1:41175/solr/collection1_shard2_replica_t4/&wt=javabin&version=2}{deleteByQuery=*:* (-1577598372346331136)} 0 0
[junit4] 2> 1296448 INFO (qtp1538454288-12170) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t4] webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1577598372346331136)} 0 16
[junit4] 2> 1296452 INFO (qtp51216966-12185) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.TestInjection Start waiting for replica in sync with leader
[junit4] 2> 1296457 INFO (qtp1538454288-12177) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1577598372372545536,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1296457 INFO (qtp1538454288-12177) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1296459 INFO (qtp51216966-12172) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1577598372374642688,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1296460 INFO (qtp51216966-12172) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1296479 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1296489 INFO (qtp1538454288-12177) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1296477 INFO (qtp1538454288-12173) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.TestInjection Start waiting for replica in sync with leader
[junit4] 2> 1296490 INFO (qtp51216966-12172) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1296490 INFO (qtp51216966-12172) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:41175/solr/collection1_shard2_replica_t4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 30
[junit4] 2> 1296491 INFO (qtp1538454288-12177) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t4] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:41175/solr/collection1_shard2_replica_t4/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 34
[junit4] 2> 1296493 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1296983 INFO (qtp1538454288-12175) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1296998 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1297487 INFO (qtp1538454288-12175) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1297502 INFO (qtp51216966-12184) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1297992 INFO (qtp1538454288-12175) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1298009 INFO (qtp51216966-12184) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1298401 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication
[junit4] 2> 1298401 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Updated masterUrl to https://127.0.0.1:41175/solr/collection1_shard2_replica_t4/
[junit4] 2> 1298402 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 1298402 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 1298402 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 1298402 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 1298402 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 1298403 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 1298403 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1298403 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1e9d8fe commitCommandVersion:0
[junit4] 2> 1298405 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.s.SolrIndexSearcher Opening [Searcher@b11b38e[collection1_shard2_replica_t6] main]
[junit4] 2> 1298405 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1298405 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication
[junit4] 2> 1298406 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Updated masterUrl to https://127.0.0.1:47999/solr/collection1_shard1_replica_t2/
[junit4] 2> 1298407 INFO (searcherExecutor-5209-thread-1-processing-n:127.0.0.1:47999_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore [collection1_shard2_replica_t6] Registered new searcher Searcher@b11b38e[collection1_shard2_replica_t6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1298407 INFO (qtp51216966-12183) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 1298408 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 1298408 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 1298408 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 1298408 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 1298408 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 1298408 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1298408 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@6c14ee96 commitCommandVersion:0
[junit4] 2> 1298410 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@66ec85e6[collection1_shard1_replica_t1] main]
[junit4] 2> 1298410 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1298411 INFO (searcherExecutor-5206-thread-1-processing-n:127.0.0.1:41175_solr x:collection1_shard1_replica_t1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore [collection1_shard1_replica_t1] Registered new searcher Searcher@66ec85e6[collection1_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1298497 INFO (qtp1538454288-12176) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1298516 INFO (qtp51216966-12184) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1299002 INFO (qtp1538454288-12176) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1299022 INFO (qtp51216966-12184) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1299506 INFO (qtp1538454288-12176) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1299527 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1300010 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1300031 INFO (qtp51216966-12184) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1300515 INFO (qtp1538454288-12174) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1300537 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1301020 INFO (qtp1538454288-12175) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1301042 INFO (qtp51216966-12184) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1301401 INFO (qtp1538454288-12170) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 1301401 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 1301401 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 1301401 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's generation: 2
[junit4] 2> 1301401 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's version: 1504515051282
[junit4] 2> 1301405 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 1301405 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 1301405 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 1301405 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's generation: 2
[junit4] 2> 1301405 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's version: 1504515051287
[junit4] 2> 1301524 INFO (qtp1538454288-12175) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1301559 INFO (qtp51216966-12184) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1302029 INFO (qtp1538454288-12170) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1302064 INFO (qtp51216966-12183) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1302534 INFO (qtp1538454288-12170) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1302568 INFO (qtp51216966-12184) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1303038 INFO (qtp1538454288-12170) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1303073 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1303542 INFO (qtp1538454288-12170) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1303577 INFO (qtp51216966-12183) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1304046 INFO (qtp1538454288-12170) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1304082 INFO (qtp51216966-12183) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1304400 INFO (qtp1538454288-12175) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 1304400 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 1304400 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 1304400 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's generation: 2
[junit4] 2> 1304400 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's version: 1504515051282
[junit4] 2> 1304404 INFO (qtp51216966-12172) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 1304404 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 1304404 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 1304404 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's generation: 2
[junit4] 2> 1304404 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's version: 1504515051287
[junit4] 2> 1304551 INFO (qtp1538454288-12175) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1304587 INFO (qtp51216966-12183) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1305055 INFO (qtp1538454288-12170) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1305091 INFO (qtp51216966-12184) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1305575 INFO (qtp1538454288-12176) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1305598 INFO (qtp51216966-12182) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1306089 INFO (qtp1538454288-12170) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1306116 INFO (qtp51216966-12184) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1306593 INFO (qtp1538454288-12170) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1306621 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1307098 INFO (qtp1538454288-12170) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1307129 INFO (qtp51216966-12183) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1307400 INFO (qtp1538454288-12175) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 1307400 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 1307400 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 1307400 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's generation: 2
[junit4] 2> 1307400 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Slave's version: 1504515051282
[junit4] 2> 1307403 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 1307403 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 1307403 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 1307403 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's generation: 2
[junit4] 2> 1307403 INFO (indexFetcher-5229-thread-1) [n:127.0.0.1:41175_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_t1] o.a.s.h.IndexFetcher Slave's version: 1504515051287
[junit4] 2> 1307602 INFO (qtp1538454288-12176) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1307634 INFO (qtp51216966-12183) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1308106 INFO (qtp1538454288-12176) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1308139 INFO (qtp51216966-12186) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1308612 INFO (qtp1538454288-12176) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1308644 INFO (qtp51216966-12182) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1309116 INFO (qtp1538454288-12176) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1309151 INFO (qtp51216966-12172) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1309621 INFO (qtp1538454288-12176) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1309661 INFO (qtp51216966-12181) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1310124 INFO (qtp1538454288-12176) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1310165 INFO (qtp51216966-12183) [n:127.0.0.1:47999_solr c:collection1 s:shard1 r:core_node5 x:collection1_shard1_replica_t2] o.a.s.c.S.Request [collection1_shard1_replica_t2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1310398 INFO (qtp1538454288-12177) [n:127.0.0.1:41175_solr c:collection1 s:shard2 r:core_node7 x:collection1_shard2_replica_t4] o.a.s.c.S.Request [collection1_shard2_replica_t4] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 1310398 INFO (indexFetcher-5227-thread-1) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 1310398 INFO (indexFetcher-52
[...truncated too long message...]
_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[false]
[junit4] 2> 1469535 INFO (recoveryExecutor-2594-thread-2-processing-n:127.0.0.1:47999_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.SolrCore [collection1_shard2_replica_t6] CLOSING SolrCore org.apache.solr.core.SolrCore@2e4af862
[junit4] 2> 1469535 INFO (recoveryExecutor-2594-thread-2-processing-n:127.0.0.1:47999_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard2.replica_t6, tag=776665186
[junit4] 2> 1469535 INFO (recoveryExecutor-2594-thread-2-processing-n:127.0.0.1:47999_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5e53a043: rootName = solr_47999, domain = solr.core.collection1.shard2.replica_t6, service url = null, agent id = null] for registry solr.core.collection1.shard2.replica_t6 / com.codahale.metrics.MetricRegistry@296dffe9
[junit4] 2> 1469561 INFO (recoveryExecutor-2594-thread-2-processing-n:127.0.0.1:47999_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard2.leader, tag=776665186
[junit4] 2> 1469561 WARN (recoveryExecutor-2594-thread-2-processing-n:127.0.0.1:47999_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1_shard2_replica_t6] coreNodeName=[core_node8]
[junit4] 2> 1469564 WARN (updateExecutor-2593-thread-1-processing-n:127.0.0.1:47999_solr x:collection1_shard2_replica_t6 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:47999_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t6] o.a.s.u.DefaultSolrCoreState Skipping recovery because Solr is shutdown
[junit4] 2> 1469565 INFO (jetty-closer-2560-thread-1) [ ] o.a.s.c.Overseer Overseer (id=98599897982500881-127.0.0.1:47999_solr-n_0000000004) closing
[junit4] 2> 1469565 INFO (OverseerStateUpdate-98599897982500881-127.0.0.1:47999_solr-n_0000000004) [n:127.0.0.1:47999_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:47999_solr
[junit4] 2> 1469571 WARN (zkCallback-2596-thread-2-processing-n:127.0.0.1:47999_solr) [n:127.0.0.1:47999_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1469571 INFO (jetty-closer-2560-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@68732343{/solr,null,UNAVAILABLE}
[junit4] 2> 1469831 WARN (zkCallback-2602-thread-2-processing-n:127.0.0.1:41175_solr) [n:127.0.0.1:41175_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1469832 INFO (jetty-closer-2560-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@501b34e3{/solr,null,UNAVAILABLE}
[junit4] 2> 1469836 ERROR (SUITE-TestCloudRecovery-seed#[B940B2629F487985]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1469837 INFO (SUITE-TestCloudRecovery-seed#[B940B2629F487985]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:51205 51205
[junit4] 2> 1479899 INFO (Thread-2428) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:51205 51205
[junit4] 2> 1479901 WARN (Thread-2428) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/aliases.json
[junit4] 2> 6 /solr/security.json
[junit4] 2> 6 /solr/configs/config
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 135 /solr/collections/collection1/state.json
[junit4] 2> 7 /solr/clusterstate.json
[junit4] 2> 7 /solr/clusterprops.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/collections
[junit4] 2> 4 /solr/overseer/queue
[junit4] 2> 4 /solr/overseer/collection-queue-work
[junit4] 2> 3 /solr/live_nodes
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudRecovery_B940B2629F487985-001
[junit4] 2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=10778, maxDocsPerChunk=6, blockSize=6), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=10778, blockSize=6)), sim=RandomSimilarity(queryNorm=true): {}, locale=ko, timezone=Turkey
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=294787336,total=515899392
[junit4] 2> NOTE: All tests run in this JVM: [TestReRankQParserPlugin, RollingRestartTest, HdfsChaosMonkeySafeLeaderTest, SearchHandlerTest, TestPerFieldSimilarity, TestEmbeddedSolrServerSchemaAPI, AliasIntegrationTest, DistributedDebugComponentTest, SolrMetricsIntegrationTest, SharedFSAutoReplicaFailoverTest, ConjunctionSolrSpellCheckerTest, OverseerStatusTest, SolrCmdDistributorTest, TestRangeQuery, TestSimpleTrackingShardHandler, TestSortByMinMaxFunction, TestSolrCloudWithDelegationTokens, TestDocumentBuilder, TestSuggestSpellingConverter, TestSolrCloudWithHadoopAuthPlugin, SolrGraphiteReporterTest, FileUtilsTest, DebugComponentTest, FullHLLTest, TestManagedResource, CloneFieldUpdateProcessorFactoryTest, SolrCloudReportersTest, SynonymTokenizerTest, TestZkChroot, LeaderElectionIntegrationTest, TestLRUStatsCache, ShufflingReplicaListTransformerTest, PreAnalyzedFieldManagedSchemaCloudTest, SuggestComponentContextFilterQueryTest, QueryResultKeyTest, SolrIndexConfigTest, BasicZkTest, TestInfoStreamLogging, TriLevelCompositeIdRoutingTest, TestPartialUpdateDeduplication, TestInitQParser, TestEmbeddedSolrServerAdminHandler, CloudExitableDirectoryReaderTest, BadComponentTest, TestNRTOpen, TestSolr4Spatial, TestDelegationWithHadoopAuth, BlockJoinFacetSimpleTest, TestDefaultStatsCache, TestSolrQueryParser, DistributedQueryElevationComponentTest, CachingDirectoryFactoryTest, TestSolrConfigHandlerCloud, TestUseDocValuesAsStored, EnumFieldTest, TestAuthenticationFramework, TestSlowCompositeReaderWrapper, TestHttpShardHandlerFactory, ShowFileRequestHandlerTest, BaseCdcrDistributedZkTest, TestTlogReplica, CursorMarkTest, TestCrossCoreJoin, BlockJoinFacetDistribTest, SegmentsInfoRequestHandlerTest, HdfsSyncSliceTest, DistributedIntervalFacetingTest, DistributedTermsComponentTest, TestMacroExpander, TestObjectReleaseTracker, TestFileDictionaryLookup, TestPayloadScoreQParserPlugin, PolyFieldTest, ConnectionManagerTest, SampleTest, TestRealTimeGet, OpenExchangeRatesOrgProviderTest, SOLR749Test, DeleteLastCustomShardedReplicaTest, TestManagedSynonymFilterFactory, TestBinaryResponseWriter, TestMultiWordSynonyms, TestClusterStateMutator, TestBM25SimilarityFactory, LeaderFailoverAfterPartitionTest, TestSimpleQParserPlugin, TestSubQueryTransformerCrossCore, TestPolicyCloud, MetricsConfigTest, UtilsToolTest, OpenCloseCoreStressTest, OverriddenZkACLAndCredentialsProvidersTest, ConvertedLegacyTest, HLLUtilTest, TestRebalanceLeaders, TestFastLRUCache, HttpSolrCallGetCoreTest, TestFiltering, TestTrie, TestDynamicFieldResource, TestCopyFieldCollectionResource, TestLegacyField, TestJsonRequest, SpellPossibilityIteratorTest, TestComplexPhraseQParserPlugin, TestExceedMaxTermLength, LukeRequestHandlerTest, JsonLoaderTest, SolrIndexMetricsTest, TestRandomFaceting, TestManagedSchemaThreadSafety, MoveReplicaHDFSTest, RuleEngineTest, ZkStateWriterTest, TestExclusionRuleCollectionAccess, SuggesterTSTTest, BigEndianAscendingWordDeserializerTest, TestDocTermOrds, TestCollapseQParserPlugin, SolrPluginUtilsTest, TestFieldSortValues, TestConfigsApi, TestMinMaxOnMultiValuedField, AddSchemaFieldsUpdateProcessorFactoryTest, TestScoreJoinQPNoScore, TestIntervalFaceting, TestShortCircuitedRequests, TermVectorComponentDistributedTest, TestSolrCloudSnapshots, DistribDocExpirationUpdateProcessorTest, PropertiesRequestHandlerTest, TestRandomFlRTGCloud, ClassificationUpdateProcessorIntegrationTest, TestCollationField, RequestLoggingTest, TestShardHandlerFactory, TestChildDocTransformer, TestPostingsSolrHighlighter, TestMultiValuedNumericRangeQuery, ExternalFileFieldSortTest, StatsComponentTest, TestSchemaResource, TestConfigSetProperties, SpatialRPTFieldTypeTest, DeleteStatusTest, TestAuthorizationFramework, HdfsBasicDistributedZkTest, ZkFailoverTest, TestCollationFieldDocValues, ChaosMonkeyNothingIsSafeWithPullReplicasTest, InfixSuggestersTest, SimpleMLTQParserTest, DataDrivenBlockJoinTest, TestClusterProperties, TestXmlQParser, TestCloudRecovery]
[junit4] Completed [302/731 (1!)] on J1 in 187.90s, 2 tests, 1 failure <<< FAILURES!
[...truncated 47708 lines...]
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 154 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/154/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC
2 tests failed.
FAILED: org.apache.solr.cloud.CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster
Error Message:
Document mismatch on target after sync expected:<2000> but was:<1100>
Stack Trace:
java.lang.AssertionError: Document mismatch on target after sync expected:<2000> but was:<1100>
at __randomizedtesting.SeedInfo.seed([917F44F251A9D77A:453A0FABB6FF6481]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.apache.solr.cloud.CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster(CdcrBootstrapTest.java:309)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 1) Thread[id=29388, name=jetty-launcher-4232-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) 2) Thread[id=29392, name=jetty-launcher-4232-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
1) Thread[id=29388, name=jetty-launcher-4232-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
2) Thread[id=29392, name=jetty-launcher-4232-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
at __randomizedtesting.SeedInfo.seed([917F44F251A9D77A]:0)
Build Log:
[...truncated 12726 lines...]
[junit4] Suite: org.apache.solr.cloud.CdcrBootstrapTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrBootstrapTest_917F44F251A9D77A-001/init-core-data-001
[junit4] 2> 1982366 WARN (SUITE-CdcrBootstrapTest-seed#[917F44F251A9D77A]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
[junit4] 2> 1982366 INFO (SUITE-CdcrBootstrapTest-seed#[917F44F251A9D77A]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 1982367 INFO (SUITE-CdcrBootstrapTest-seed#[917F44F251A9D77A]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 1982367 INFO (SUITE-CdcrBootstrapTest-seed#[917F44F251A9D77A]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 1982369 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[917F44F251A9D77A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testBootstrapWithSourceCluster
[junit4] 2> 1982369 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[917F44F251A9D77A]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrBootstrapTest_917F44F251A9D77A-001/cdcr-target-001
[junit4] 2> 1982370 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[917F44F251A9D77A]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1982370 INFO (Thread-7268) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1982370 INFO (Thread-7268) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1982371 ERROR (Thread-7268) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1982472 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[917F44F251A9D77A]) [ ] o.a.s.c.ZkTestServer start zk server on port:58194
[junit4] 2> 1982478 INFO (jetty-launcher-3068-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1982479 INFO (jetty-launcher-3068-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4b8cb232{/solr,null,AVAILABLE}
[junit4] 2> 1982481 INFO (jetty-launcher-3068-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@491be669{HTTP/1.1,[http/1.1]}{127.0.0.1:59747}
[junit4] 2> 1982481 INFO (jetty-launcher-3068-thread-1) [ ] o.e.j.s.Server Started @1985139ms
[junit4] 2> 1982481 INFO (jetty-launcher-3068-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59747}
[junit4] 2> 1982481 ERROR (jetty-launcher-3068-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1982481 INFO (jetty-launcher-3068-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.1.0
[junit4] 2> 1982481 INFO (jetty-launcher-3068-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1982481 INFO (jetty-launcher-3068-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1982481 INFO (jetty-launcher-3068-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-03T21:45:47.407Z
[junit4] 2> 1982483 INFO (jetty-launcher-3068-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1982490 INFO (jetty-launcher-3068-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58194/solr
[junit4] 2> 1982621 INFO (jetty-launcher-3068-thread-1) [n:127.0.0.1:59747_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1982621 INFO (jetty-launcher-3068-thread-1) [n:127.0.0.1:59747_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59747_solr
[junit4] 2> 1982623 INFO (jetty-launcher-3068-thread-1) [n:127.0.0.1:59747_solr ] o.a.s.c.Overseer Overseer (id=98597283253256195-127.0.0.1:59747_solr-n_0000000000) starting
[junit4] 2> 1982631 INFO (jetty-launcher-3068-thread-1) [n:127.0.0.1:59747_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59747_solr
[junit4] 2> 1982634 INFO (zkCallback-3075-thread-1-processing-n:127.0.0.1:59747_solr) [n:127.0.0.1:59747_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1982738 INFO (jetty-launcher-3068-thread-1) [n:127.0.0.1:59747_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59747.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@279a6fef
[junit4] 2> 1982747 INFO (jetty-launcher-3068-thread-1) [n:127.0.0.1:59747_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59747.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@279a6fef
[junit4] 2> 1982747 INFO (jetty-launcher-3068-thread-1) [n:127.0.0.1:59747_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59747.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@279a6fef
[junit4] 2> 1982748 INFO (jetty-launcher-3068-thread-1) [n:127.0.0.1:59747_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrBootstrapTest_917F44F251A9D77A-001/cdcr-target-001/node1/.
[junit4] 1> Target zkHost = 127.0.0.1:58194/solr
[junit4] 2> 1982768 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[917F44F251A9D77A]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrBootstrapTest_917F44F251A9D77A-001/cdcr-source-001
[junit4] 2> 1982769 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[917F44F251A9D77A]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1982769 INFO (Thread-7278) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1982769 INFO (Thread-7278) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1982770 ERROR (Thread-7278) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1982869 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[917F44F251A9D77A]) [ ] o.a.s.c.ZkTestServer start zk server on port:60830
[junit4] 2> 1982877 INFO (jetty-launcher-3080-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1982878 INFO (jetty-launcher-3080-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7b508611{/solr,null,AVAILABLE}
[junit4] 2> 1982878 INFO (jetty-launcher-3080-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@c5d6b91{HTTP/1.1,[http/1.1]}{127.0.0.1:62658}
[junit4] 2> 1982878 INFO (jetty-launcher-3080-thread-1) [ ] o.e.j.s.Server Started @1985537ms
[junit4] 2> 1982878 INFO (jetty-launcher-3080-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=62658}
[junit4] 2> 1982879 ERROR (jetty-launcher-3080-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1982879 INFO (jetty-launcher-3080-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.1.0
[junit4] 2> 1982879 INFO (jetty-launcher-3080-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1982879 INFO (jetty-launcher-3080-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1982879 INFO (jetty-launcher-3080-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-09-03T21:45:47.805Z
[junit4] 2> 1982881 INFO (jetty-launcher-3080-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1982886 INFO (jetty-launcher-3080-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60830/solr
[junit4] 2> 1983013 INFO (jetty-launcher-3080-thread-1) [n:127.0.0.1:62658_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1983014 INFO (jetty-launcher-3080-thread-1) [n:127.0.0.1:62658_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:62658_solr
[junit4] 2> 1983015 INFO (jetty-launcher-3080-thread-1) [n:127.0.0.1:62658_solr ] o.a.s.c.Overseer Overseer (id=98597283279405059-127.0.0.1:62658_solr-n_0000000000) starting
[junit4] 2> 1983025 INFO (jetty-launcher-3080-thread-1) [n:127.0.0.1:62658_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62658_solr
[junit4] 2> 1983026 INFO (OverseerStateUpdate-98597283279405059-127.0.0.1:62658_solr-n_0000000000) [n:127.0.0.1:62658_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1983078 INFO (jetty-launcher-3080-thread-1) [n:127.0.0.1:62658_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62658.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@279a6fef
[junit4] 2> 1983083 INFO (jetty-launcher-3080-thread-1) [n:127.0.0.1:62658_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62658.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@279a6fef
[junit4] 2> 1983083 INFO (jetty-launcher-3080-thread-1) [n:127.0.0.1:62658_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62658.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@279a6fef
[junit4] 2> 1983084 INFO (jetty-launcher-3080-thread-1) [n:127.0.0.1:62658_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrBootstrapTest_917F44F251A9D77A-001/cdcr-source-001/node1/.
[junit4] 2> 1983119 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[917F44F251A9D77A]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1983120 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[917F44F251A9D77A]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60830/solr ready
[junit4] 2> 1983123 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=cdcr-source&name=cdcr-source&nrtReplicas=1&action=CREATE&numShards=1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1983126 INFO (OverseerThreadFactory-8584-thread-1-processing-n:127.0.0.1:62658_solr) [n:127.0.0.1:62658_solr ] o.a.s.c.CreateCollectionCmd Create collection cdcr-source
[junit4] 2> 1983238 INFO (OverseerStateUpdate-98597283279405059-127.0.0.1:62658_solr-n_0000000000) [n:127.0.0.1:62658_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-source",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"cdcr-source_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:62658/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 1983440 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=cdcr-source&newCollection=true&collection=cdcr-source&version=2&replicaType=NRT&coreNodeName=core_node2&name=cdcr-source_shard1_replica_n1&action=CREATE&numShards=1&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
[junit4] 2> 1983440 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1983546 INFO (zkCallback-3087-thread-1-processing-n:127.0.0.1:62658_solr) [n:127.0.0.1:62658_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1983546 INFO (zkCallback-3087-thread-2-processing-n:127.0.0.1:62658_solr) [n:127.0.0.1:62658_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1984461 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 1984464 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
[junit4] 2> 1984467 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.IndexSchema [cdcr-source_shard1_replica_n1] Schema name=minimal
[junit4] 2> 1984468 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1984469 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Created and persisted managed schema znode at /configs/cdcr-source/managed-schema
[junit4] 2> 1984474 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, renamed the non-managed schema /configs/cdcr-source/schema.xml to /configs/cdcr-source/schema.xml.bak
[junit4] 2> 1984474 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-source_shard1_replica_n1' using configuration from collection cdcr-source, trusted=true
[junit4] 2> 1984474 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62658.solr.core.cdcr-source.shard1.replica_n1' (registry 'solr.core.cdcr-source.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@279a6fef
[junit4] 2> 1984474 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1984474 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [[cdcr-source_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrBootstrapTest_917F44F251A9D77A-001/cdcr-source-001/node1/cdcr-source_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrBootstrapTest_917F44F251A9D77A-001/cdcr-source-001/node1/./cdcr-source_shard1_replica_n1/data/]
[junit4] 2> 1984523 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 1984523 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1984524 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1984524 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1984526 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@139d11a8[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 1984529 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-source
[junit4] 2> 1984529 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-source
[junit4] 2> 1984529 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-source/managed-schema
[junit4] 2> 1984530 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 1984531 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1984534 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrBufferStateManager Created znode /collections/cdcr-source/cdcr/state/buffer
[junit4] 2> 1984536 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrProcessStateManager Created znode /collections/cdcr-source/cdcr/state/process
[junit4] 2> 1984538 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577556534324363264
[junit4] 2> 1984539 INFO (searcherExecutor-8587-thread-1-processing-n:127.0.0.1:62658_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@139d11a8[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1984545 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1984545 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1984545 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:62658/solr/cdcr-source_shard1_replica_n1/
[junit4] 2> 1984546 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1984546 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:62658/solr/cdcr-source_shard1_replica_n1/ has no replicas
[junit4] 2> 1984546 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1984549 INFO (zkCallback-3087-thread-2-processing-n:127.0.0.1:62658_solr) [n:127.0.0.1:62658_solr ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-source:shard1
[junit4] 2> 1984551 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:62658/solr/cdcr-source_shard1_replica_n1/ shard1
[junit4] 2> 1984661 INFO (zkCallback-3087-thread-1-processing-n:127.0.0.1:62658_solr) [n:127.0.0.1:62658_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1984661 INFO (zkCallback-3087-thread-2-processing-n:127.0.0.1:62658_solr) [n:127.0.0.1:62658_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1984705 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1984707 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=cdcr-source&newCollection=true&collection=cdcr-source&version=2&replicaType=NRT&coreNodeName=core_node2&name=cdcr-source_shard1_replica_n1&action=CREATE&numShards=1&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=1267
[junit4] 2> 1984710 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1984810 INFO (zkCallback-3087-thread-1-processing-n:127.0.0.1:62658_solr) [n:127.0.0.1:62658_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1984810 INFO (zkCallback-3087-thread-2-processing-n:127.0.0.1:62658_solr) [n:127.0.0.1:62658_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1985129 INFO (OverseerCollectionConfigSetProcessor-98597283279405059-127.0.0.1:62658_solr-n_0000000000) [n:127.0.0.1:62658_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1985710 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=cdcr-source&name=cdcr-source&nrtReplicas=1&action=CREATE&numShards=1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2} status=0 QTime=2588
[junit4] 1> Adding 10 docs with commit=true, numDocs=100
[junit4] 2> 1985717 INFO (qtp490479419-22091) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 1985718 INFO (qtp490479419-22092) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1577556535561682944,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1985718 INFO (qtp490479419-22092) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@6792bceb commitCommandVersion:1577556535561682944
[junit4] 2> 1985728 INFO (qtp490479419-22092) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7b194411[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 1985728 INFO (qtp490479419-22092) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1985729 INFO (searcherExecutor-8587-thread-1-processing-n:127.0.0.1:62658_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@7b194411[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100)))}
[junit4] 2> 1985729 INFO (qtp490479419-22092) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=11
[junit4] 1> Adding 10 docs with commit=true, numDocs=200
[junit4] 2> 1985733 INFO (qtp490479419-22093) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 1985734 INFO (qtp490479419-22086) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1577556535578460160,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1985734 INFO (qtp490479419-22086) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@6792bceb commitCommandVersion:1577556535578460160
[junit4] 2> 1985743 INFO (qtp490479419-22086) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5309f8d8[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 1985744 INFO (qtp490479419-22086) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1985744 INFO (searcherExecutor-8587-thread-1-processing-n:127.0.0.1:62658_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@5309f8d8[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100)))}
[junit4] 2> 1985744 INFO (qtp490479419-22086) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=10
[junit4] 1> Adding 10 docs with commit=true, numDocs=300
[junit4] 2> 1985751 INFO (qtp490479419-22089) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=5
[junit4] 2> 1985751 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1577556535596285952,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1985751 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@6792bceb commitCommandVersion:1577556535596285952
[junit4] 2> 1985763 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@33d4331d[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 1985763 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1985764 INFO (searcherExecutor-8587-thread-1-processing-n:127.0.0.1:62658_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@33d4331d[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100)))}
[junit4] 2> 1985764 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=13
[junit4] 1> Adding 10 docs with commit=true, numDocs=400
[junit4] 2> 1985768 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1985768 INFO (qtp490479419-22091) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1577556535614111752,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1985769 INFO (qtp490479419-22091) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@6792bceb commitCommandVersion:1577556535614111752
[junit4] 2> 1985780 INFO (qtp490479419-22091) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@46c7f01[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 1985781 INFO (qtp490479419-22091) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1985782 INFO (searcherExecutor-8587-thread-1-processing-n:127.0.0.1:62658_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@46c7f01[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100) Uninverting(_3(7.1.0):C100)))}
[junit4] 2> 1985782 INFO (qtp490479419-22091) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=13
[junit4] 1> Adding 10 docs with commit=true, numDocs=500
[junit4] 2> 1985785 INFO (qtp490479419-22092) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 1985786 INFO (qtp490479419-22093) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1577556535632986112,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1985786 INFO (qtp490479419-22093) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@6792bceb commitCommandVersion:1577556535632986112
[junit4] 2> 1985798 INFO (qtp490479419-22093) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@ac73bf2[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 1985798 INFO (qtp490479419-22093) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1985799 INFO (searcherExecutor-8587-thread-1-processing-n:127.0.0.1:62658_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@ac73bf2[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100) Uninverting(_3(7.1.0):C100) Uninverting(_4(7.1.0):C100)))}
[junit4] 2> 1985799 INFO (qtp490479419-22093) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=13
[junit4] 1> Adding 10 docs with commit=true, numDocs=600
[junit4] 2> 1985802 INFO (qtp490479419-22086) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1985803 INFO (qtp490479419-22089) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1577556535650811904,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1985803 INFO (qtp490479419-22089) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@6792bceb commitCommandVersion:1577556535650811904
[junit4] 2> 1985814 INFO (qtp490479419-22089) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@227bf76c[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 1985815 INFO (qtp490479419-22089) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1985815 INFO (searcherExecutor-8587-thread-1-processing-n:127.0.0.1:62658_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@227bf76c[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100) Uninverting(_3(7.1.0):C100) Uninverting(_4(7.1.0):C100) Uninverting(_5(7.1.0):C100)))}
[junit4] 2> 1985816 INFO (qtp490479419-22089) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=12
[junit4] 1> Adding 10 docs with commit=true, numDocs=700
[junit4] 2> 1985819 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1985819 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1577556535667589135,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1985819 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@6792bceb commitCommandVersion:1577556535667589135
[junit4] 2> 1985833 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7607643c[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 1985834 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1985835 INFO (searcherExecutor-8587-thread-1-processing-n:127.0.0.1:62658_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@7607643c[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100) Uninverting(_3(7.1.0):C100) Uninverting(_4(7.1.0):C100) Uninverting(_5(7.1.0):C100) Uninverting(_6(7.1.0):C100)))}
[junit4] 2> 1985835 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=15
[junit4] 1> Adding 10 docs with commit=true, numDocs=800
[junit4] 2> 1985839 INFO (qtp490479419-22091) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 1985839 INFO (qtp490479419-22092) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1577556535688560651,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1985839 INFO (qtp490479419-22092) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@6792bceb commitCommandVersion:1577556535688560651
[junit4] 2> 1985852 INFO (qtp490479419-22092) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@26632fbf[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 1985852 INFO (qtp490479419-22092) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1985853 INFO (searcherExecutor-8587-thread-1-processing-n:127.0.0.1:62658_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@26632fbf[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100) Uninverting(_3(7.1.0):C100) Uninverting(_4(7.1.0):C100) Uninverting(_5(7.1.0):C100) Uninverting(_6(7.1.0):C100) Uninverting(_7(7.1.0):C100)))}
[junit4] 2> 1985853 INFO (qtp490479419-22092) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=14
[junit4] 1> Adding 10 docs with commit=true, numDocs=900
[junit4] 2> 1985857 INFO (qtp490479419-22093) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1985857 INFO (qtp490479419-22086) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1577556535707435019,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1985858 INFO (qtp490479419-22086) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@6792bceb commitCommandVersion:1577556535707435019
[junit4] 2> 1985873 INFO (qtp490479419-22086) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@675c2a0f[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 1985873 INFO (qtp490479419-22086) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1985874 INFO (searcherExecutor-8587-thread-1-processing-n:127.0.0.1:62658_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@675c2a0f[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100) Uninverting(_3(7.1.0):C100) Uninverting(_4(7.1.0):C100) Uninverting(_5(7.1.0):C100) Uninverting(_6(7.1.0):C100) Uninverting(_7(7.1.0):C100) Uninverting(_8(7.1.0):C100)))}
[junit4] 2> 1985874 INFO (qtp490479419-22086) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=17
[junit4] 1> Adding 10 docs with commit=true, numDocs=1000
[junit4] 2> 1985878 INFO (qtp490479419-22089) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1985878 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1577556535729455116,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1985878 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@6792bceb commitCommandVersion:1577556535729455116
[junit4] 2> 1985895 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3d300fc1[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 1985895 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1985897 INFO (searcherExecutor-8587-thread-1-processing-n:127.0.0.1:62658_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@3d300fc1[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.1.0):C100) Uninverting(_1(7.1.0):C100) Uninverting(_2(7.1.0):C100) Uninverting(_3(7.1.0):C100) Uninverting(_4(7.1.0):C100) Uninverting(_5(7.1.0):C100) Uninverting(_6(7.1.0):C100) Uninverting(_7(7.1.0):C100) Uninverting(_8(7.1.0):C100) Uninverting(_9(7.1.0):C100)))}
[junit4] 2> 1985897 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=18
[junit4] 2> 1985898 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&_stateVer_=cdcr-source:4&wt=javabin&version=2} hits=1000 status=0 QTime=0
[junit4] 2> 1985904 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e49b4ec210006, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1985905 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[917F44F251A9D77A]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1985906 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[917F44F251A9D77A]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:58194/solr ready
[junit4] 2> 1985908 INFO (qtp1778856406-22047) [n:127.0.0.1:59747_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=cdcr-target&name=cdcr-target&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1985915 INFO (OverseerThreadFactory-8576-thread-1-processing-n:127.0.0.1:59747_solr) [n:127.0.0.1:59747_solr ] o.a.s.c.CreateCollectionCmd Create collection cdcr-target
[junit4] 2> 1986026 INFO (OverseerStateUpdate-98597283253256195-127.0.0.1:59747_solr-n_0000000000) [n:127.0.0.1:59747_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-target",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"cdcr-target_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:59747/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 1986232 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=cdcr-target&newCollection=true&name=cdcr-target_shard1_replica_n1&action=CREATE&numShards=1&collection=cdcr-target&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1986232 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1986344 INFO (zkCallback-3075-thread-1-processing-n:127.0.0.1:59747_solr) [n:127.0.0.1:59747_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1986344 INFO (zkCallback-3075-thread-2-processing-n:127.0.0.1:59747_solr) [n:127.0.0.1:59747_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1987248 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 1987251 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
[junit4] 2> 1987253 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.IndexSchema [cdcr-target_shard1_replica_n1] Schema name=minimal
[junit4] 2> 1987254 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1987255 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Created and persisted managed schema znode at /configs/cdcr-target/managed-schema
[junit4] 2> 1987258 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, renamed the non-managed schema /configs/cdcr-target/schema.xml to /configs/cdcr-target/schema.xml.bak
[junit4] 2> 1987258 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-target_shard1_replica_n1' using configuration from collection cdcr-target, trusted=true
[junit4] 2> 1987259 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59747.solr.core.cdcr-target.shard1.replica_n1' (registry 'solr.core.cdcr-target.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@279a6fef
[junit4] 2> 1987259 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1987259 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SolrCore [[cdcr-target_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrBootstrapTest_917F44F251A9D77A-001/cdcr-target-001/node1/cdcr-target_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrBootstrapTest_917F44F251A9D77A-001/cdcr-target-001/node1/./cdcr-target_shard1_replica_n1/data/]
[junit4] 2> 1987300 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 1987300 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1987301 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1987301 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1987302 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3c2176d2[cdcr-target_shard1_replica_n1] main]
[junit4] 2> 1987303 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-target
[junit4] 2> 1987304 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-target
[junit4] 2> 1987304 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-target/managed-schema
[junit4] 2> 1987304 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 1987305 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1987308 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.CdcrBufferStateManager Created znode /collections/cdcr-target/cdcr/state/buffer
[junit4] 2> 1987309 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.CdcrProcessStateManager Created znode /collections/cdcr-target/cdcr/state/process
[junit4] 2> 1987310 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577556537231015936
[junit4] 2> 1987311 INFO (searcherExecutor-8593-thread-1-processing-n:127.0.0.1:59747_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-target_shard1_replica_n1] Registered new searcher Searcher@3c2176d2[cdcr-target_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1987316 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1987316 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1987316 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:59747/solr/cdcr-target_shard1_replica_n1/
[junit4] 2> 1987316 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1987316 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:59747/solr/cdcr-target_shard1_replica_n1/ has no replicas
[junit4] 2> 1987316 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1987320 INFO (zkCallback-3075-thread-2-processing-n:127.0.0.1:59747_solr) [n:127.0.0.1:59747_solr ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-target:shard1
[junit4] 2> 1987322 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:59747/solr/cdcr-target_shard1_replica_n1/ shard1
[junit4] 2> 1987428 INFO (zkCallback-3075-thread-2-processing-n:127.0.0.1:59747_solr) [n:127.0.0.1:59747_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1987428 INFO (zkCallback-3075-thread-1-processing-n:127.0.0.1:59747_solr) [n:127.0.0.1:59747_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1987483 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1987485 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=cdcr-target&newCollection=true&name=cdcr-target_shard1_replica_n1&action=CREATE&numShards=1&collection=cdcr-target&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1253
[junit4] 2> 1987488 INFO (qtp1778856406-22047) [n:127.0.0.1:59747_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1987589 INFO (zkCallback-3075-thread-1-processing-n:127.0.0.1:59747_solr) [n:127.0.0.1:59747_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1987589 INFO (zkCallback-3075-thread-2-processing-n:127.0.0.1:59747_solr) [n:127.0.0.1:59747_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1987925 INFO (OverseerCollectionConfigSetProcessor-98597283253256195-127.0.0.1:59747_solr-n_0000000000) [n:127.0.0.1:59747_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1988489 INFO (qtp1778856406-22047) [n:127.0.0.1:59747_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=cdcr-target&name=cdcr-target&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2582
[junit4] 2> 1988494 INFO (qtp1778856406-22049) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.S.Request [cdcr-target_shard1_replica_n1] webapp=/solr path=/cdcr params={qt=/cdcr&_stateVer_=cdcr-target:4&action=start&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 1988495 INFO (zkCallback-3075-thread-2-processing-n:127.0.0.1:59747_solr) [n:127.0.0.1:59747_solr ] o.a.s.h.CdcrProcessStateManager The CDCR process state has changed: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/cdcr/state/process @ cdcr-target:shard1
[junit4] 2> 1988496 INFO (zkCallback-3075-thread-2-processing-n:127.0.0.1:59747_solr) [n:127.0.0.1:59747_solr ] o.a.s.h.CdcrProcessStateManager Received new CDCR process state from watcher: STARTED @ cdcr-target:shard1
[junit4] 2> 1988499 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1988501 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:58194/solr ready
[junit4] 2> 1988509 INFO (qtp1778856406-22050) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.CdcrRequestHandler Found maxVersionFromRecent 0 maxVersionFromIndex 0
[junit4] 2> 1988509 INFO (qtp1778856406-22050) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.S.Request [cdcr-target_shard1_replica_n1] webapp=/solr path=/cdcr params={action=SHARDCHECKPOINT&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1988510 INFO (qtp1778856406-22052) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.S.Request [cdcr-target_shard1_replica_n1] webapp=/solr path=/cdcr params={_stateVer_=cdcr-target:4&action=COLLECTIONCHECKPOINT&wt=javabin&version=2} status=0 QTime=7
[junit4] 2> 1988510 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrReplicatorManager Create new update log reader for target cdcr-target with checkpoint -1 @ cdcr-source:shard1
[junit4] 2> 1988510 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrReplicatorManager Attempting to bootstrap target collection: cdcr-target, shard: shard1
[junit4] 2> 1988516 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrReplicatorManager Submitting bootstrap task to executor
[junit4] 2> 1988517 INFO (zkCallback-3087-thread-2-processing-n:127.0.0.1:62658_solr) [n:127.0.0.1:62658_solr ] o.a.s.h.CdcrProcessStateManager The CDCR process state has changed: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/cdcr/state/process @ cdcr-source:shard1
[junit4] 2> 1988517 INFO (zkCallback-3087-thread-2-processing-n:127.0.0.1:62658_solr) [n:127.0.0.1:62658_solr ] o.a.s.h.CdcrProcessStateManager Received new CDCR process state from watcher: STARTED @ cdcr-source:shard1
[junit4] 2> 1988517 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/cdcr params={qt=/cdcr&_stateVer_=cdcr-source:4&action=start&wt=javabin&version=2} status=0 QTime=22
[junit4] 2> 1988518 INFO (cdcr-bootstrap-status-3099-thread-1-processing-n:127.0.0.1:62658_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrReplicatorManager Attempting to bootstrap target collection: cdcr-target shard: shard1 leader: http://127.0.0.1:59747/solr/cdcr-target_shard1_replica_n1/
[junit4] 2> 1988521 INFO (qtp490479419-22091) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/cdcr params={qt=/cdcr&_stateVer_=cdcr-source:4&action=queues&wt=javabin&version=2} status=0 QTime=2
[junit4] 1> Cdcr queue response: {responseHeader={status=0,QTime=2},queues={127.0.0.1:58194/solr={cdcr-target={queueSize=1010,lastTimestamp=}}},tlogTotalSize=47659,tlogTotalCount=10,updateLogSynchronizer=stopped}
[junit4] 2> 1988521 INFO (qtp1778856406-22048) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.S.Request [cdcr-target_shard1_replica_n1] webapp=/solr path=/cdcr params={qt=/cdcr&masterUrl=http://127.0.0.1:62658/solr/cdcr-source_shard1_replica_n1/&action=BOOTSTRAP&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1988523 INFO (qtp1778856406-22049) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1577556538502938624,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1988523 INFO (qtp1778856406-22049) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1988524 INFO (qtp1778856406-22049) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1988524 INFO (qtp1778856406-22049) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.S.Request [cdcr-target_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-target:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 1988525 INFO (recoveryExecutor-3073-thread-1-processing-n:127.0.0.1:59747_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 1988525 INFO (qtp1778856406-22051) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.S.Request [cdcr-target_shard1_replica_n1] webapp=/solr path=/cdcr params={qt=/cdcr&action=BOOTSTRAP_STATUS&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1988526 INFO (qtp1778856406-22051) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.S.Request [cdcr-target_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&_stateVer_=cdcr-target:4&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 1988526 INFO (cdcr-bootstrap-status-3099-thread-1-processing-n:127.0.0.1:62658_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrReplicatorManager CDCR bootstrap running for 1 seconds, sleeping for 2000 ms
[junit4] 2> 1988527 INFO (qtp490479419-22093) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1577556538507132928,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1988527 INFO (qtp490479419-22093) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@6792bceb commitCommandVersion:1577556538507132928
[junit4] 2> 1988533 INFO (qtp490479419-22093) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6a560f0a[cdcr-source_shard1_replica_n1] realtime]
[junit4] 2> 1988534 INFO (qtp490479419-22093) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1988534 INFO (qtp490479419-22093) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&commit_end_point=true&wt=javabin&version=2} status=0 QTime=7
[junit4] 2> 1988535 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 1988536 INFO (recoveryExecutor-3073-thread-1-processing-n:127.0.0.1:59747_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Master's generation: 12
[junit4] 2> 1988536 INFO (recoveryExecutor-3073-thread-1-processing-n:127.0.0.1:59747_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Master's version: 1504475153453
[junit4] 2> 1988536 INFO (recoveryExecutor-3073-thread-1-processing-n:127.0.0.1:59747_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 1988536 INFO (recoveryExecutor-3073-thread-1-processing-n:127.0.0.1:59747_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 1988536 INFO (recoveryExecutor-3073-thread-1-processing-n:127.0.0.1:59747_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Starting replication process
[junit4] 2> 1988540 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.ReplicationHandler Adding tlog files to list: [{size=4657, name=tlog.0000000000000000000.1577556535556440064}, {size=4778, name=tlog.0000000000000000001.1577556535574265856}, {size=4778, name=tlog.0000000000000000002.1577556535589994496}, {size=4778, name=tlog.0000000000000000003.1577556535610966016}, {size=4778, name=tlog.0000000000000000004.1577556535629840384}, {size=4778, name=tlog.0000000000000000005.1577556535647666176}, {size=4778, name=tlog.0000000000000000006.1577556535665491968}, {size=4778, name=tlog.0000000000000000007.1577556535685414912}, {size=4778, name=tlog.0000000000000000008.1577556535704289280}, {size=4778, name=tlog.0000000000000000009.1577556535726309376}]
[junit4] 2> 1988540 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&wt=javabin&version=2&command=filelist} status=0 QTime=3
[junit4] 2> 1988540 INFO (recoveryExecutor-3073-thread-1-processing-n:127.0.0.1:59747_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Number of files in latest index in master: 13
[junit4] 2> 1988541 INFO (recoveryExecutor-3073-thread-1-processing-n:127.0.0.1:59747_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Number of tlog files in master: 10
[junit4] 2> 1988541 INFO (recoveryExecutor-3073-thread-1-processing-n:127.0.0.1:59747_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 1988542 INFO (recoveryExecutor-3073-thread-1-processing-n:127.0.0.1:59747_solr x:cdcr-target_shard1_replica_n1 s:shard1 c:cdcr-target r:core_node2) [n:127.0.0.1:59747_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.h.IndexFetcher Starting download (fullCopy=false) to MockDirectoryWrapper(RAMDirectory@76e49d57 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@470b678e)
[junit4] 2> 1988542 INFO (qtp490479419-22091) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1988544 INFO (qtp490479419-22092) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a_Asserting_0.pos&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1988546 INFO (qtp490479419-22093) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1988547 INFO (qtp490479419-22089) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1988548 INFO (qtp490479419-22086) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a_Asserting_0.tip&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1988549 INFO (qtp490479419-22090) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/replication params={generation=12&qt=/replication&file=_a_Asserting_0.tim&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1988550 INFO (qtp490479419-22088) [n:127.0.0.1:62658_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shar
[...truncated too long message...]
rocessorCloud, DisMaxRequestHandlerTest, HdfsBasicDistributedZk2Test, DirectUpdateHandlerTest, TestBackupRepositoryFactory, TestSurroundQueryParser, TestPseudoReturnFields, TestEmbeddedSolrServerConstructors, CoreAdminCreateDiscoverTest, SolrCoreMetricManagerTest, ClusterStateTest, TestCodecSupport, TestMissingGroups, TestCloudPseudoReturnFields, GraphQueryTest, TestLocalFSCloudBackupRestore, SpatialFilterTest, TestDistribIDF, NoCacheHeaderTest, TestExpandComponent, TestReloadAndDeleteDocs, TestLegacyNumericRangeQueryBuilder, TestRawResponseWriter, PeerSyncWithIndexFingerprintCachingTest, TestLazyCores, ReturnFieldsTest, HdfsUnloadDistributedZkTest, UnloadDistributedZkTest, TestTestInjection, MinimalSchemaTest, TestXmlQParserPlugin, TestManagedResourceStorage, TestManagedSchemaAPI, TestLeaderElectionZkExpiry, TestComplexPhraseLeadingWildcard, SimpleFacetsTest, SolrTestCaseJ4Test, ReplicaListTransformerTest, TestSolrDeletionPolicy2, TestBulkSchemaConcurrent, NumericFieldsTest, ZkControllerTest, OverseerCollectionConfigSetProcessorTest, RegexBoostProcessorTest, TestJettySolrRunner, WordBreakSolrSpellCheckerTest, TestSweetSpotSimilarityFactory, DirectoryFactoryTest, BasicAuthIntegrationTest, DateFieldTest, TestInPlaceUpdatesStandalone, TestGeoJSONResponseWriter, TestMacros, ChaosMonkeyNothingIsSafeTest, TestRecoveryHdfs, ActionThrottleTest, TestStressRecovery, V2ApiIntegrationTest, TlogReplayBufferedWhileIndexingTest, ForceLeaderTest, TestReplicaProperties, TestLegacyFieldReuse, TestPullReplicaErrorHandling, TestStressCloudBlindAtomicUpdates, SortByFunctionTest, CdcrVersionReplicationTest, TestLFUCache, TestNumericTokenStream, ShardRoutingTest, TestZkAclsWithHadoopAuth, TestXIncludeConfig, MultiTermTest, HdfsDirectoryFactoryTest, ClassificationUpdateProcessorFactoryTest, SoftAutoCommitTest, TestNoOpRegenerator, TestFreeTextSuggestions, RAMDirectoryFactoryTest, TestFieldTypeCollectionResource, TestHalfAndHalfDocValues, LeaderInitiatedRecoveryOnShardRestartTest, TestFunctionQuery, TestHdfsCloudBackupRestore, UUIDFieldTest, LeaderFailureAfterFreshStartTest, BooleanFieldTest, TestNumericRangeQuery32, TestAddFieldRealTimeGet, TestInitParams, SecurityConfHandlerTest, TestSort, SolrJmxReporterCloudTest, RecoveryZkTest, ClusterStateUpdateTest, AnalysisAfterCoreReloadTest, TestStressReorder, TestCoreContainer, TestMiniSolrCloudClusterSSL, TestClassicSimilarityFactory, TestSearcherReuse, AnalyticsQueryTest, QueryElevationComponentTest, DocValuesTest, CoreAdminHandlerTest, DocValuesMissingTest, TestBlendedInfixSuggestions, OutputWriterTest, SolrMetricReporterTest, BJQParserTest, TestSchemaSimilarityResource, TestExtendedDismaxParser, SSLMigrationTest, CdcrUpdateLogTest, TestIBSimilarityFactory, ParsingFieldUpdateProcessorsTest, TestFieldCache, CloudMLTQParserTest, TestCustomDocTransformer, TestPKIAuthenticationPlugin, CursorPagingTest, TestSolrCoreProperties, TestLMDirichletSimilarityFactory, TestQuerySenderNoQuery, FacetPivotSmallTest, DeleteShardTest, AddBlockUpdateTest, HdfsRestartWhileUpdatingTest, SolrGangliaReporterTest, TestConfigSetImmutable, StressHdfsTest, TestIndexingPerformance, TestSchemaNameResource, TestLegacyFieldCache, TestSolrQueryResponse, ExplicitHLLTest, DefaultValueUpdateProcessorTest, HdfsThreadLeakTest, TestFaceting, StatsReloadRaceTest, TestStreamBody, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, TestReplicationHandler, ZkCLITest, TestRecovery, DistributedSpellCheckComponentTest, TestReload, DistributedTermsComponentTest, SolrCoreTest, StatsComponentTest, ConvertedLegacyTest, BasicFunctionalityTest, HighlighterTest, SimplePostToolTest, TestCoreDiscovery, SuggesterFSTTest, SolrRequestParserTest, SuggesterTest, TestStressLucene, SuggesterWFSTTest, TestAtomicUpdateErrorCases, QueryEqualityTest, DocValuesMultiTest, DistanceFunctionTest, TestSolrDeletionPolicy1, SolrInfoBeanTest, CacheHeaderTest, LukeRequestHandlerTest, PrimitiveFieldTypeTest, RequestHandlersTest, TestValueSourceCache, TermVectorComponentTest, FieldAnalysisRequestHandlerTest, FastVectorHighlighterTest, TestJmxIntegration, QueryParsingTest, JsonLoaderTest, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, TestComponentsName, SearchHandlerTest, CopyFieldTest, SolrIndexConfigTest, SampleTest, TestDocSet, TestSearchPerf, ExternalFileFieldSortTest, NotRequiredUniqueKeyTest, TestPhraseSuggestions, TestBM25SimilarityFactory, TimeZoneUtilsTest, ScriptEngineTest, PreAnalyzedFieldTest, TestSuggestSpellingConverter, DOMUtilTest, ZkNodePropsTest, CircularListTest, TestCursorMarkWithoutUniqueKey, TestHighlightDedupGrouping, AsyncCallRequestStatusResponseTest, CdcrBootstrapTest, CollectionsAPISolrJTest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, DistributedQueueTest, DocValuesNotIndexedTest, LeaderFailoverAfterPartitionTest, MigrateRouteKeyTest, MoveReplicaHDFSUlogDirTest, MultiThreadedOCPTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerTaskQueueTest, ReplaceNodeTest, ShardSplitTest, TestConfigSetsAPIExclusivity, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestDistribDocBasedVersion, TestDownShardTolerantSearch, TestExclusionRuleCollectionAccess, TestLeaderInitiatedRecoveryThread, TestPullReplica, TestSegmentSorting, TestSizeLimitedDistributedMap, TestSolrCloudWithSecureImpersonation]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=917F44F251A9D77A -Dtests.slow=true -Dtests.locale=he -Dtests.timezone=Asia/Saigon -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | TestSolrCloudWithSecureImpersonation (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
[junit4] > 1) Thread[id=29388, name=jetty-launcher-4232-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] > at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] > at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] > at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] > at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] > 2) Thread[id=29392, name=jetty-launcher-4232-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] > at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] > at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] > at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] > at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] > at __randomizedtesting.SeedInfo.seed([917F44F251A9D77A]:0)
[junit4] Completed [562/731 (2!)] on J0 in 49.67s, 12 tests, 1 error <<< FAILURES!
[...truncated 46835 lines...]