You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2017/10/15 07:10:12 UTC
[JENKINS] Lucene-Solr-Tests-7.x - Build # 183 - Still Unstable
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/183/
1 tests failed.
FAILED: org.apache.solr.cloud.TestHdfsCloudBackupRestore.test
Error Message:
expected:<{shard1=0, shard2=3}> but was:<{shard1=0, shard2=0}>
Stack Trace:
java.lang.AssertionError: expected:<{shard1=0, shard2=3}> but was:<{shard1=0, shard2=0}>
at __randomizedtesting.SeedInfo.seed([1A52D2B00C0736A5:9206ED6AA2FB5B5D]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:285)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12061 lines...]
[junit4] Suite: org.apache.solr.cloud.TestHdfsCloudBackupRestore
[junit4] 2> 1221422 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/init-core-data-001
[junit4] 2> 1221423 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 1221440 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 1221441 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 1221616 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 1221667 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1221668 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1221810 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_lucene2.us.west_apache_org_39242_hdfs____.lmp18s/webapp
[junit4] 2> 1223386 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@lucene2-us-west.apache.org:39242
[junit4] 2> 1224005 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1224005 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1224097 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_35942_datanode____.ca2284/webapp
[junit4] 2> 1225046 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35942
[junit4] 2> 1225224 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 1225225 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 1225262 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_40134_datanode____.6f56mb/webapp
[junit4] 2> 1225464 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to lucene2-us-west.apache.org/127.0.0.1:43133) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 1225517 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x1cf5c5e94521b6: from storage DS-051ac79c-83e4-4b11-ba0d-e9500bdddff4 node DatanodeRegistration(127.0.0.1:44003, datanodeUuid=fca6dce5-6985-4811-9b92-edf8040f3e0f, infoPort=46286, infoSecurePort=0, ipcPort=41438, storageInfo=lv=-56;cid=testClusterID;nsid=254902223;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 1225517 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x1cf5c5e94521b6: from storage DS-8a34a8d3-b413-4700-abfd-5b1a74cb71cc node DatanodeRegistration(127.0.0.1:44003, datanodeUuid=fca6dce5-6985-4811-9b92-edf8040f3e0f, infoPort=46286, infoSecurePort=0, ipcPort=41438, storageInfo=lv=-56;cid=testClusterID;nsid=254902223;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 1226612 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40134
[junit4] 2> 1227475 ERROR (DataNode: [[[DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to lucene2-us-west.apache.org/127.0.0.1:43133) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 1227501 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x1cf5c6608fe121: from storage DS-98bcdee0-a66f-413d-bc94-d1f8be5116cf node DatanodeRegistration(127.0.0.1:33596, datanodeUuid=618d2365-3371-42a3-9dfa-32f051628dd0, infoPort=43743, infoSecurePort=0, ipcPort=45511, storageInfo=lv=-56;cid=testClusterID;nsid=254902223;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 1227501 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x1cf5c6608fe121: from storage DS-b7143b2c-9597-4d25-a630-6bb600c802b1 node DatanodeRegistration(127.0.0.1:33596, datanodeUuid=618d2365-3371-42a3-9dfa-32f051628dd0, infoPort=43743, infoSecurePort=0, ipcPort=45511, storageInfo=lv=-56;cid=testClusterID;nsid=254902223;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 1227620 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.s.c.TestHdfsCloudBackupRestore The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 1232631 WARN (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.s.c.TestHdfsCloudBackupRestore The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 1237676 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-002
[junit4] 2> 1237676 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1237690 INFO (Thread-1751) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1237690 INFO (Thread-1751) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1237755 ERROR (Thread-1751) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1237804 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:38810
[junit4] 2> 1237930 INFO (jetty-launcher-572-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1237959 INFO (jetty-launcher-572-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@39a136dd{/solr,null,AVAILABLE}
[junit4] 2> 1237959 INFO (jetty-launcher-572-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@f84d064{HTTP/1.1,[http/1.1]}{127.0.0.1:33650}
[junit4] 2> 1237959 INFO (jetty-launcher-572-thread-1) [ ] o.e.j.s.Server Started @1250812ms
[junit4] 2> 1237959 INFO (jetty-launcher-572-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33650}
[junit4] 2> 1237993 ERROR (jetty-launcher-572-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1237993 INFO (jetty-launcher-572-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 1237993 INFO (jetty-launcher-572-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1237993 INFO (jetty-launcher-572-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1237993 INFO (jetty-launcher-572-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-15T04:37:12.934Z
[junit4] 2> 1238011 INFO (jetty-launcher-572-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1238030 INFO (jetty-launcher-572-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5144a051{/solr,null,AVAILABLE}
[junit4] 2> 1238030 INFO (jetty-launcher-572-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3ca679a{HTTP/1.1,[http/1.1]}{127.0.0.1:46340}
[junit4] 2> 1238030 INFO (jetty-launcher-572-thread-2) [ ] o.e.j.s.Server Started @1250883ms
[junit4] 2> 1238030 INFO (jetty-launcher-572-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46340}
[junit4] 2> 1238030 ERROR (jetty-launcher-572-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1238030 INFO (jetty-launcher-572-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 1238031 INFO (jetty-launcher-572-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1238031 INFO (jetty-launcher-572-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1238031 INFO (jetty-launcher-572-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-15T04:37:12.972Z
[junit4] 2> 1238065 INFO (jetty-launcher-572-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1238085 INFO (jetty-launcher-572-thread-2) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@14749a65, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1238093 INFO (jetty-launcher-572-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1238096 INFO (jetty-launcher-572-thread-1) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@14749a65, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1238113 INFO (jetty-launcher-572-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38810/solr
[junit4] 2> 1238137 INFO (jetty-launcher-572-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38810/solr
[junit4] 2> 1238646 INFO (jetty-launcher-572-thread-1) [n:127.0.0.1:33650_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1238647 INFO (jetty-launcher-572-thread-1) [n:127.0.0.1:33650_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33650_solr
[junit4] 2> 1238648 INFO (jetty-launcher-572-thread-1) [n:127.0.0.1:33650_solr ] o.a.s.c.Overseer Overseer (id=98831055757770757-127.0.0.1:33650_solr-n_0000000000) starting
[junit4] 2> 1238725 INFO (jetty-launcher-572-thread-1) [n:127.0.0.1:33650_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33650_solr
[junit4] 2> 1238771 INFO (zkCallback-583-thread-1-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1238929 INFO (jetty-launcher-572-thread-1) [n:127.0.0.1:33650_solr ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = hdfs,class = org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = {name=hdfs, class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = {location=/backup,solr.hdfs.home=hdfs://lucene2-us-west.apache.org:43133/solr,solr.hdfs.confdir=}}
[junit4] 2> 1238929 INFO (jetty-launcher-572-thread-1) [n:127.0.0.1:33650_solr ] o.a.s.c.b.r.BackupRepositoryFactory Default configuration for backup repository is with configuration params {type = repository,name = hdfs,class = org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = {name=hdfs, class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = {location=/backup,solr.hdfs.home=hdfs://lucene2-us-west.apache.org:43133/solr,solr.hdfs.confdir=}}
[junit4] 2> 1239020 INFO (jetty-launcher-572-thread-2) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1239021 INFO (jetty-launcher-572-thread-2) [n:127.0.0.1:46340_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1239023 INFO (jetty-launcher-572-thread-2) [n:127.0.0.1:46340_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46340_solr
[junit4] 2> 1239109 INFO (zkCallback-584-thread-1-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1239126 INFO (jetty-launcher-572-thread-1) [n:127.0.0.1:33650_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14749a65
[junit4] 2> 1239132 INFO (jetty-launcher-572-thread-1) [n:127.0.0.1:33650_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14749a65
[junit4] 2> 1239132 INFO (jetty-launcher-572-thread-1) [n:127.0.0.1:33650_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14749a65
[junit4] 2> 1239133 INFO (jetty-launcher-572-thread-1) [n:127.0.0.1:33650_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-002/node1/.
[junit4] 2> 1239171 INFO (zkCallback-583-thread-2-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1239324 INFO (jetty-launcher-572-thread-2) [n:127.0.0.1:46340_solr ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = hdfs,class = org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = {name=hdfs, class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = {location=/backup,solr.hdfs.home=hdfs://lucene2-us-west.apache.org:43133/solr,solr.hdfs.confdir=}}
[junit4] 2> 1239337 INFO (jetty-launcher-572-thread-2) [n:127.0.0.1:46340_solr ] o.a.s.c.b.r.BackupRepositoryFactory Default configuration for backup repository is with configuration params {type = repository,name = hdfs,class = org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = {name=hdfs, class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = {location=/backup,solr.hdfs.home=hdfs://lucene2-us-west.apache.org:43133/solr,solr.hdfs.confdir=}}
[junit4] 2> 1239434 INFO (jetty-launcher-572-thread-2) [n:127.0.0.1:46340_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14749a65
[junit4] 2> 1239579 INFO (jetty-launcher-572-thread-2) [n:127.0.0.1:46340_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14749a65
[junit4] 2> 1239580 INFO (jetty-launcher-572-thread-2) [n:127.0.0.1:46340_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14749a65
[junit4] 2> 1239630 INFO (jetty-launcher-572-thread-2) [n:127.0.0.1:46340_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-002/node2/.
[junit4] 2> 1239931 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1239932 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38810/solr ready
[junit4] 2> 1240831 INFO (TEST-TestHdfsCloudBackupRestore.test-seed#[1A52D2B00C0736A5]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 1240834 INFO (qtp1120325267-4889) [n:127.0.0.1:46340_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&router.name=implicit&version=2&pullReplicas=0&shards=shard1,shard2&property.customKey=customValue&maxShardsPerNode=2&router.field=shard_s&autoAddReplicas=true&name=hdfsbackuprestore&nrtReplicas=1&action=CREATE&tlogReplicas=1&wt=javabin and sendToOCPQueue=true
[junit4] 2> 1240868 INFO (OverseerThreadFactory-1624-thread-1-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.CreateCollectionCmd Create collection hdfsbackuprestore
[junit4] 2> 1241058 INFO (OverseerStateUpdate-98831055757770757-127.0.0.1:33650_solr-n_0000000000) [n:127.0.0.1:33650_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"hdfsbackuprestore_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:46340/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 1241059 INFO (OverseerStateUpdate-98831055757770757-127.0.0.1:33650_solr-n_0000000000) [n:127.0.0.1:33650_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"hdfsbackuprestore_shard1_replica_t2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:33650/solr",
[junit4] 2> "type":"TLOG"}
[junit4] 2> 1241060 INFO (OverseerStateUpdate-98831055757770757-127.0.0.1:33650_solr-n_0000000000) [n:127.0.0.1:33650_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"hdfsbackuprestore_shard2_replica_n3",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:46340/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 1241074 INFO (OverseerStateUpdate-98831055757770757-127.0.0.1:33650_solr-n_0000000000) [n:127.0.0.1:33650_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"hdfsbackuprestore",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"hdfsbackuprestore_shard2_replica_t7",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:33650/solr",
[junit4] 2> "type":"TLOG"}
[junit4] 2> 1241339 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node4&name=hdfsbackuprestore_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 1241339 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1241340 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node8&name=hdfsbackuprestore_shard2_replica_t7&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 1241340 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1241341 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node5&name=hdfsbackuprestore_shard1_replica_t2&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 1241465 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node6&name=hdfsbackuprestore_shard2_replica_n3&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 1241648 INFO (zkCallback-583-thread-2-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1241648 INFO (zkCallback-583-thread-1-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1241664 INFO (zkCallback-584-thread-1-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1241664 INFO (zkCallback-583-thread-3-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1241701 INFO (zkCallback-584-thread-2-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1241725 INFO (zkCallback-584-thread-3-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1242516 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 1242591 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.s.IndexSchema [hdfsbackuprestore_shard1_replica_n1] Schema name=minimal
[junit4] 2> 1242629 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 1242642 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1242642 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_shard1_replica_n1' using configuration from collection hdfsbackuprestore, trusted=true
[junit4] 2> 1242643 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore.shard1.replica_n1' (registry 'solr.core.hdfsbackuprestore.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14749a65
[junit4] 2> 1242643 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1242643 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.SolrCore [[hdfsbackuprestore_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-002/node2/hdfsbackuprestore_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-002/node2/./hdfsbackuprestore_shard1_replica_n1/data/]
[junit4] 2> 1242647 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.s.IndexSchema [hdfsbackuprestore_shard2_replica_n3] Schema name=minimal
[junit4] 2> 1242661 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1242661 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_shard2_replica_n3' using configuration from collection hdfsbackuprestore, trusted=true
[junit4] 2> 1242662 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore.shard2.replica_n3' (registry 'solr.core.hdfsbackuprestore.shard2.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14749a65
[junit4] 2> 1242662 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1242662 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.SolrCore [[hdfsbackuprestore_shard2_replica_n3] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-002/node2/hdfsbackuprestore_shard2_replica_n3], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-002/node2/./hdfsbackuprestore_shard2_replica_n3/data/]
[junit4] 2> 1242698 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 1242701 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 1242883 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.s.IndexSchema [hdfsbackuprestore_shard2_replica_t7] Schema name=minimal
[junit4] 2> 1242918 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.s.IndexSchema [hdfsbackuprestore_shard1_replica_t2] Schema name=minimal
[junit4] 2> 1242918 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1242918 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_shard2_replica_t7' using configuration from collection hdfsbackuprestore, trusted=true
[junit4] 2> 1242918 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore.shard2.replica_t7' (registry 'solr.core.hdfsbackuprestore.shard2.replica_t7') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14749a65
[junit4] 2> 1242918 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1242918 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.c.SolrCore [[hdfsbackuprestore_shard2_replica_t7] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-002/node1/hdfsbackuprestore_shard2_replica_t7], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-002/node1/./hdfsbackuprestore_shard2_replica_t7/data/]
[junit4] 2> 1242919 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1242919 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_shard1_replica_t2' using configuration from collection hdfsbackuprestore, trusted=true
[junit4] 2> 1242920 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore.shard1.replica_t2' (registry 'solr.core.hdfsbackuprestore.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14749a65
[junit4] 2> 1242920 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1242920 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.c.SolrCore [[hdfsbackuprestore_shard1_replica_t2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-002/node1/hdfsbackuprestore_shard1_replica_t2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-002/node1/./hdfsbackuprestore_shard1_replica_t2/data/]
[junit4] 2> 1243193 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1243193 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1243194 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1243195 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1243196 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@536ffc2e[hdfsbackuprestore_shard2_replica_n3] main]
[junit4] 2> 1243230 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1243230 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1243231 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1243231 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581296897933443072
[junit4] 2> 1243283 INFO (searcherExecutor-1630-thread-1-processing-n:127.0.0.1:46340_solr x:hdfsbackuprestore_shard2_replica_n3 s:shard2 c:hdfsbackuprestore r:core_node6) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.SolrCore [hdfsbackuprestore_shard2_replica_n3] Registered new searcher Searcher@536ffc2e[hdfsbackuprestore_shard2_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1243319 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1243319 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1243320 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1243320 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1243342 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=9999ms
[junit4] 2> 1243392 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6fce71af[hdfsbackuprestore_shard1_replica_n1] main]
[junit4] 2> 1243393 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1243393 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1243394 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1243394 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581296898104360960
[junit4] 2> 1243503 INFO (zkCallback-583-thread-1-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1243505 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 1243518 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1243518 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1243519 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1243519 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1243521 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@5e1cb8a4[hdfsbackuprestore_shard1_replica_t2] main]
[junit4] 2> 1243539 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1243539 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1243540 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1243540 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581296898257453056
[junit4] 2> 1243577 INFO (searcherExecutor-1629-thread-1-processing-n:127.0.0.1:46340_solr x:hdfsbackuprestore_shard1_replica_n1 s:shard1 c:hdfsbackuprestore r:core_node4) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.SolrCore [hdfsbackuprestore_shard1_replica_n1] Registered new searcher Searcher@6fce71af[hdfsbackuprestore_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1243578 INFO (searcherExecutor-1632-thread-1-processing-n:127.0.0.1:33650_solr x:hdfsbackuprestore_shard1_replica_t2 s:shard1 c:hdfsbackuprestore r:core_node5) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.c.SolrCore [hdfsbackuprestore_shard1_replica_t2] Registered new searcher Searcher@5e1cb8a4[hdfsbackuprestore_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1243592 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1243592 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1243593 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1243593 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1243594 INFO (zkCallback-583-thread-3-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1243594 INFO (zkCallback-583-thread-1-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1243594 INFO (zkCallback-584-thread-2-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1243594 INFO (zkCallback-584-thread-1-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1243596 INFO (zkCallback-584-thread-3-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1243665 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.s.SolrIndexSearcher Opening [Searcher@56a59427[hdfsbackuprestore_shard2_replica_t7] main]
[junit4] 2> 1243671 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1243672 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1243672 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1243673 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581296898395865088
[junit4] 2> 1243723 INFO (searcherExecutor-1631-thread-1-processing-n:127.0.0.1:33650_solr x:hdfsbackuprestore_shard2_replica_t7 s:shard2 c:hdfsbackuprestore r:core_node8) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.c.SolrCore [hdfsbackuprestore_shard2_replica_t7] Registered new searcher Searcher@56a59427[hdfsbackuprestore_shard2_replica_t7] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1243724 INFO (zkCallback-584-thread-2-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1243724 INFO (zkCallback-584-thread-1-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1243724 INFO (zkCallback-584-thread-3-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1243724 INFO (zkCallback-583-thread-3-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1243724 INFO (zkCallback-583-thread-4-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1243724 INFO (zkCallback-583-thread-1-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1243848 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1243848 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1243848 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:46340/solr/hdfsbackuprestore_shard2_replica_n3/
[junit4] 2> 1243849 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.u.PeerSync PeerSync: core=hdfsbackuprestore_shard2_replica_n3 url=http://127.0.0.1:46340/solr START replicas=[http://127.0.0.1:33650/solr/hdfsbackuprestore_shard2_replica_t7/] nUpdates=100
[junit4] 2> 1243921 INFO (qtp1306448183-4879) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_t7] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=19
[junit4] 2> 1244011 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1244011 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1244011 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:46340/solr/hdfsbackuprestore_shard1_replica_n1/
[junit4] 2> 1244011 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=hdfsbackuprestore_shard1_replica_n1 url=http://127.0.0.1:46340/solr START replicas=[http://127.0.0.1:33650/solr/hdfsbackuprestore_shard1_replica_t2/] nUpdates=100
[junit4] 2> 1244026 INFO (qtp1306448183-4880) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.c.S.Request [hdfsbackuprestore_shard1_replica_t2] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=14
[junit4] 2> 1244173 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.u.PeerSync PeerSync: core=hdfsbackuprestore_shard2_replica_n3 url=http://127.0.0.1:46340/solr DONE. We have no versions. sync failed.
[junit4] 2> 1244173 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 1244173 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 1244173 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1244175 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:46340/solr/hdfsbackuprestore_shard2_replica_n3/ shard2
[junit4] 2> 1244319 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=hdfsbackuprestore_shard1_replica_n1 url=http://127.0.0.1:46340/solr DONE. We have no versions. sync failed.
[junit4] 2> 1244319 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 1244319 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 1244319 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1244321 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:46340/solr/hdfsbackuprestore_shard1_replica_n1/ shard1
[junit4] 2> 1244506 INFO (zkCallback-583-thread-1-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1244520 INFO (zkCallback-583-thread-3-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1244520 INFO (zkCallback-583-thread-4-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1244522 INFO (zkCallback-584-thread-2-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1244522 INFO (zkCallback-584-thread-3-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1244522 INFO (zkCallback-584-thread-1-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1244538 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1244560 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1244561 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node4&name=hdfsbackuprestore_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin} status=0 QTime=3222
[junit4] 2> 1244562 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node6&name=hdfsbackuprestore_shard2_replica_n3&action=CREATE&numShards=2&shard=shard2&wt=javabin} status=0 QTime=3097
[junit4] 2> 1244613 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.c.ZkController hdfsbackuprestore_shard1_replica_t2 starting background replication from leader
[junit4] 2> 1244613 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 1244648 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 1244648 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1244649 INFO (qtp1306448183-4875) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node5&name=hdfsbackuprestore_shard1_replica_t2&action=CREATE&numShards=2&shard=shard1&wt=javabin} status=0 QTime=3307
[junit4] 2> 1244795 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.c.ZkController hdfsbackuprestore_shard2_replica_t7 starting background replication from leader
[junit4] 2> 1244795 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 1244961 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 1244961 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1244962 INFO (qtp1306448183-4876) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=hdfsbackuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node8&name=hdfsbackuprestore_shard2_replica_t7&action=CREATE&numShards=2&shard=shard2&wt=javabin} status=0 QTime=3622
[junit4] 2> 1244990 INFO (zkCallback-584-thread-1-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1244990 INFO (zkCallback-584-thread-3-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1244990 INFO (zkCallback-584-thread-2-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1244998 INFO (zkCallback-583-thread-3-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1244998 INFO (zkCallback-583-thread-4-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1244998 INFO (zkCallback-583-thread-1-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1244999 INFO (indexFetcher-1652-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:46340/solr/hdfsbackuprestore_shard2_replica_n3/
[junit4] 2> 1245015 INFO (qtp1120325267-4893) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 1245015 INFO (indexFetcher-1652-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 1245015 INFO (indexFetcher-1652-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 1245015 INFO (indexFetcher-1652-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 1245015 INFO (indexFetcher-1652-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 1245015 INFO (indexFetcher-1652-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 1245162 INFO (indexFetcher-1652-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.s.SolrIndexSearcher Opening [Searcher@6c3d22e6[hdfsbackuprestore_shard2_replica_t7] main]
[junit4] 2> 1245163 INFO (searcherExecutor-1631-thread-1-processing-n:127.0.0.1:33650_solr x:hdfsbackuprestore_shard2_replica_t7 s:shard2 c:hdfsbackuprestore r:core_node8) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.c.SolrCore [hdfsbackuprestore_shard2_replica_t7] Registered new searcher Searcher@6c3d22e6[hdfsbackuprestore_shard2_replica_t7] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1245173 INFO (qtp1120325267-4889) [n:127.0.0.1:46340_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1245384 INFO (zkCallback-583-thread-1-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1245384 INFO (zkCallback-583-thread-4-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1245384 INFO (zkCallback-583-thread-3-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1245400 INFO (zkCallback-584-thread-2-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1245400 INFO (zkCallback-584-thread-3-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1245400 INFO (zkCallback-584-thread-1-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/hdfsbackuprestore/state.json] for collection [hdfsbackuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1245929 INFO (indexFetcher-1650-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:46340/solr/hdfsbackuprestore_shard1_replica_n1/
[junit4] 2> 1245930 INFO (qtp1120325267-4893) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.S.Request [hdfsbackuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 1245930 INFO (indexFetcher-1650-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 1245930 INFO (indexFetcher-1650-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 1245930 INFO (indexFetcher-1650-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 1245930 INFO (indexFetcher-1650-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 1245930 INFO (indexFetcher-1650-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 1245932 INFO (indexFetcher-1650-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@147ca3c[hdfsbackuprestore_shard1_replica_t2] main]
[junit4] 2> 1245987 INFO (searcherExecutor-1632-thread-1-processing-n:127.0.0.1:33650_solr x:hdfsbackuprestore_shard1_replica_t2 s:shard1 c:hdfsbackuprestore r:core_node5) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.c.SolrCore [hdfsbackuprestore_shard1_replica_t2] Registered new searcher Searcher@147ca3c[hdfsbackuprestore_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1246185 INFO (qtp1120325267-4889) [n:127.0.0.1:46340_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&router.name=implicit&version=2&pullReplicas=0&shards=shard1,shard2&property.customKey=customValue&maxShardsPerNode=2&router.field=shard_s&autoAddReplicas=true&name=hdfsbackuprestore&nrtReplicas=1&action=CREATE&tlogReplicas=1&wt=javabin} status=0 QTime=5350
[junit4] 2> 1246305 INFO (qtp1306448183-4881) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.u.p.LogUpdateProcessorFactory [hdfsbackuprestore_shard2_replica_t7] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46340/solr/hdfsbackuprestore_shard2_replica_n3/&wt=javabin&version=2}{add=[0 (1581296901071831040), 1 (1581296901124259840), 2 (1581296901125308416)]} 0 10
[junit4] HEARTBEAT J2 PID(18457@lucene2-us-west.apache.org): 2017-10-15T04:37:46, stalled for 63.7s at: ForceLeaderTest.testLastPublishedStateIsActive
[junit4] 2> 1246311 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.u.p.LogUpdateProcessorFactory [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:46340/solr/hdfsbackuprestore_shard1_replica_n1/&wt=javabin&version=2}{add=[0 (1581296901071831040), 1 (1581296901124259840), 2 (1581296901125308416)]} 0 86
[junit4] 2> 1246315 INFO (qtp1120325267-4890) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [hdfsbackuprestore_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[0, 1, 2]} 0 93
[junit4] 2> 1246406 INFO (qtp1120325267-4892) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581296901262671872,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1246406 INFO (qtp1120325267-4893) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581296901262671872,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1246413 INFO (qtp1306448183-4882) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.u.TestInjection Start waiting for replica in sync with leader
[junit4] 2> 1246415 INFO (qtp1306448183-4879) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.u.TestInjection Start waiting for replica in sync with leader
[junit4] 2> 1246681 INFO (qtp1120325267-4892) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1246681 INFO (qtp1120325267-4893) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@5eed693c commitCommandVersion:1581296901262671872
[junit4] 2> 1246881 INFO (qtp1120325267-4892) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1246881 INFO (qtp1120325267-4892) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [hdfsbackuprestore_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:46340/solr/hdfsbackuprestore_shard2_replica_n3/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 475
[junit4] 2> 1246934 INFO (qtp1120325267-4892) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=37
[junit4] 2> 1246935 INFO (qtp1120325267-4889) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard1 r:core_node4 x:hdfsbackuprestore_shard1_replica_n1] o.a.s.c.S.Request [hdfsbackuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=274
[junit4] 2> 1246935 INFO (qtp1306448183-4882) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [hdfsbackuprestore_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:46340/solr/hdfsbackuprestore_shard2_replica_n3/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 522
[junit4] 2> 1247449 INFO (qtp1120325267-4891) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1247482 INFO (qtp1120325267-4893) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@342ee91a[hdfsbackuprestore_shard2_replica_n3] main]
[junit4] 2> 1247484 INFO (searcherExecutor-1630-thread-1-processing-n:127.0.0.1:46340_solr x:hdfsbackuprestore_shard2_replica_n3 s:shard2 c:hdfsbackuprestore r:core_node6) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.SolrCore [hdfsbackuprestore_shard2_replica_n3] Registered new searcher Searcher@342ee91a[hdfsbackuprestore_shard2_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C3)))}
[junit4] 2> 1247485 INFO (qtp1120325267-4893) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1247485 INFO (qtp1120325267-4893) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.u.p.LogUpdateProcessorFactory [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:46340/solr/hdfsbackuprestore_shard2_replica_n3/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1078
[junit4] 2> 1247965 INFO (qtp1120325267-4892) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 1247966 INFO (indexFetcher-1652-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.IndexFetcher Master's generation: 2
[junit4] 2> 1247966 INFO (indexFetcher-1652-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.IndexFetcher Master's version: 1508042241622
[junit4] 2> 1247966 INFO (indexFetcher-1652-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 1247967 INFO (indexFetcher-1652-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 1247967 INFO (indexFetcher-1652-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.IndexFetcher Starting replication process
[junit4] 2> 1247968 INFO (qtp1120325267-4892) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={generation=2&qt=/replication&wt=javabin&version=2&command=filelist} status=0 QTime=1
[junit4] 2> 1247969 INFO (indexFetcher-1652-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.IndexFetcher Number of files in latest index in master: 18
[junit4] 2> 1247983 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 1248002 INFO (indexFetcher-1652-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.h.IndexFetcher Starting download (fullCopy=false) to MMapDirectory@/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001/tempDir-002/node1/hdfsbackuprestore_shard2_replica_t7/data/index.20171015053722910 lockFactory=org.apache.lucene.store.NativeFSLockFactory@62b57160
[junit4] 2> 1248003 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1248027 INFO (qtp1120325267-4890) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Direct_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1248101 INFO (qtp1120325267-4889) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Direct_0.tim&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1248185 INFO (qtp1120325267-4892) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_Direct_0.tip&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1248186 INFO (qtp1120325267-4893) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1248187 INFO (qtp1120325267-4887) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1248189 INFO (qtp1120325267-4890) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1248202 INFO (qtp1120325267-4890) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.dii&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1248203 INFO (qtp1120325267-4890) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_MockRandom_0.pos&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1248222 INFO (qtp1120325267-4890) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_MockRandom_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1248238 INFO (qtp1120325267-4889) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0_MockRandom_0.tim&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
[junit4] 2> 1248239 INFO (qtp1120325267-4892) [n:127.0.0.1:46340_solr c:hdfsbackuprestore s:shard2 r:core_node6 x:hdfsbackuprestore_shard2_replica_n3] o.a.s.c.S.Request [hdfsbackuprestore_shard2_replica_n3] webapp=/solr path=/replication pa
[...truncated too long message...]
sbackuprestore_restored.shard2.replica_n41, service url = null, agent id = null] for registry solr.core.hdfsbackuprestore_restored.shard2.replica_n41 / com.codahale.metrics.MetricRegistry@2a9a0264
[junit4] 2> 1264716 INFO (coreCloseExecutor-1679-thread-3) [n:127.0.0.1:33650_solr c:hdfsbackuprestore_restored s:shard2 r:core_node46 x:hdfsbackuprestore_restored_shard2_replica_t45] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.hdfsbackuprestore_restored.shard2.replica_t45, tag=412209052
[junit4] 2> 1264717 INFO (coreCloseExecutor-1679-thread-3) [n:127.0.0.1:33650_solr c:hdfsbackuprestore_restored s:shard2 r:core_node46 x:hdfsbackuprestore_restored_shard2_replica_t45] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@42391743: rootName = null, domain = solr.core.hdfsbackuprestore_restored.shard2.replica_t45, service url = null, agent id = null] for registry solr.core.hdfsbackuprestore_restored.shard2.replica_t45 / com.codahale.metrics.MetricRegistry@61ea58d1
[junit4] 2> 1264818 INFO (coreCloseExecutor-1678-thread-3) [n:127.0.0.1:46340_solr c:hdfsbackuprestore_restored s:shard2 r:core_node42 x:hdfsbackuprestore_restored_shard2_replica_n41] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.hdfsbackuprestore_restored.shard2.leader, tag=76754075
[junit4] 2> 1264818 INFO (coreCloseExecutor-1678-thread-4) [n:127.0.0.1:46340_solr c:hdfsbackuprestore_restored s:shard1 r:core_node44 x:hdfsbackuprestore_restored_shard1_replica_n43] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.hdfsbackuprestore_restored.shard1.replica_n43, tag=1896917766
[junit4] 2> 1264818 INFO (coreCloseExecutor-1678-thread-4) [n:127.0.0.1:46340_solr c:hdfsbackuprestore_restored s:shard1 r:core_node44 x:hdfsbackuprestore_restored_shard1_replica_n43] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3b9d862a: rootName = null, domain = solr.core.hdfsbackuprestore_restored.shard1.replica_n43, service url = null, agent id = null] for registry solr.core.hdfsbackuprestore_restored.shard1.replica_n43 / com.codahale.metrics.MetricRegistry@390a8d22
[junit4] 2> 1264873 INFO (coreCloseExecutor-1679-thread-3) [n:127.0.0.1:33650_solr c:hdfsbackuprestore_restored s:shard2 r:core_node46 x:hdfsbackuprestore_restored_shard2_replica_t45] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.hdfsbackuprestore_restored.shard2.leader, tag=412209052
[junit4] 2> 1264914 INFO (coreCloseExecutor-1679-thread-4) [n:127.0.0.1:33650_solr c:hdfsbackuprestore_restored s:shard1 r:core_node48 x:hdfsbackuprestore_restored_shard1_replica_t47] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.hdfsbackuprestore_restored.shard1.replica_t47, tag=1928131343
[junit4] 2> 1264914 INFO (coreCloseExecutor-1679-thread-4) [n:127.0.0.1:33650_solr c:hdfsbackuprestore_restored s:shard1 r:core_node48 x:hdfsbackuprestore_restored_shard1_replica_t47] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@c4a2e41: rootName = null, domain = solr.core.hdfsbackuprestore_restored.shard1.replica_t47, service url = null, agent id = null] for registry solr.core.hdfsbackuprestore_restored.shard1.replica_t47 / com.codahale.metrics.MetricRegistry@194b6f6a
[junit4] 2> 1264915 INFO (coreCloseExecutor-1678-thread-4) [n:127.0.0.1:46340_solr c:hdfsbackuprestore_restored s:shard1 r:core_node44 x:hdfsbackuprestore_restored_shard1_replica_n43] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.hdfsbackuprestore_restored.shard1.leader, tag=1896917766
[junit4] 2> 1264990 INFO (coreCloseExecutor-1679-thread-4) [n:127.0.0.1:33650_solr c:hdfsbackuprestore_restored s:shard1 r:core_node48 x:hdfsbackuprestore_restored_shard1_replica_t47] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.hdfsbackuprestore_restored.shard1.leader, tag=1928131343
[junit4] 2> 1264991 INFO (coreCloseExecutor-1679-thread-1) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard1 r:core_node5 x:hdfsbackuprestore_shard1_replica_t2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.hdfsbackuprestore.shard1.leader, tag=1281540121
[junit4] 2> 1264998 INFO (coreCloseExecutor-1679-thread-2) [n:127.0.0.1:33650_solr c:hdfsbackuprestore s:shard2 r:core_node8 x:hdfsbackuprestore_shard2_replica_t7] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.hdfsbackuprestore.shard2.leader, tag=1524881456
[junit4] 2> 1265093 WARN (zkCallback-584-thread-2-processing-n:127.0.0.1:46340_solr) [n:127.0.0.1:46340_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1265094 INFO (zkCallback-583-thread-3-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 1265128 INFO (jetty-closer-573-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5144a051{/solr,null,UNAVAILABLE}
[junit4] 2> 1265145 INFO (jetty-closer-573-thread-1) [ ] o.a.s.c.Overseer Overseer (id=98831055757770757-127.0.0.1:33650_solr-n_0000000000) closing
[junit4] 2> 1265145 INFO (OverseerStateUpdate-98831055757770757-127.0.0.1:33650_solr-n_0000000000) [n:127.0.0.1:33650_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:33650_solr
[junit4] 2> 1265165 WARN (OverseerAutoScalingTriggerThread-98831055757770757-127.0.0.1:33650_solr-n_0000000000) [n:127.0.0.1:33650_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 1266666 WARN (zkCallback-583-thread-3-processing-n:127.0.0.1:33650_solr) [n:127.0.0.1:33650_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1266667 INFO (jetty-closer-573-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@39a136dd{/solr,null,UNAVAILABLE}
[junit4] 2> 1266667 ERROR (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1266668 INFO (SUITE-TestHdfsCloudBackupRestore-seed#[1A52D2B00C0736A5]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38810 38810
[junit4] 2> 1266960 INFO (Thread-1751) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38810 38810
[junit4] 2> 1266960 WARN (Thread-1751) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/collections/hdfsbackuprestore_restored/state.json
[junit4] 2> 3 /solr/aliases.json
[junit4] 2> 3 /solr/clusterprops.json
[junit4] 2> 2 /solr/security.json
[junit4] 2> 2 /solr/configs/customConfigName
[junit4] 2> 2 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 38 /solr/collections/hdfsbackuprestore/state.json
[junit4] 2> 3 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/live_nodes
[junit4] 2> 3 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestHdfsCloudBackupRestore_1A52D2B00C0736A5-001
[junit4] 2> Oct 15, 2017 4:37:41 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 3 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {shard_s=PostingsFormat(name=MockRandom), id=PostingsFormat(name=Direct)}, docValues:{}, maxPointsInLeafNode=1664, maxMBSortInHeap=7.910597613868237, sim=RandomSimilarity(queryNorm=false): {}, locale=hr, timezone=Africa/Brazzaville
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=2,free=114562240,total=514850816
[junit4] 2> NOTE: All tests run in this JVM: [ZkNodePropsTest, CollectionTooManyReplicasTest, TestCollapseQParserPlugin, TestUniqueKeyFieldResource, ZkStateReaderTest, ConcurrentDeleteAndCreateCollectionTest, SparseHLLTest, SolrIndexMetricsTest, IndexBasedSpellCheckerTest, TestExpandComponent, ClassificationUpdateProcessorIntegrationTest, TestFieldCacheSortRandom, TestSolrXml, TestSubQueryTransformerCrossCore, TestSchemaManager, TestFieldCollectionResource, BigEndianAscendingWordSerializerTest, TestMiniSolrCloudClusterSSL, DistributedSuggestComponentTest, NodeMutatorTest, DocExpirationUpdateProcessorFactoryTest, SSLMigrationTest, MetricsConfigTest, TestRealTimeGet, DeleteLastCustomShardedReplicaTest, HdfsRecoveryZkTest, TestFieldTypeResource, TestRangeQuery, HdfsThreadLeakTest, RankQueryTest, SolrCoreTest, TestEmbeddedSolrServerSchemaAPI, SolrMetricReporterTest, TestFaceting, TestCSVResponseWriter, FullHLLTest, HdfsCollectionsAPIDistributedZkTest, TestManagedSchemaAPI, DistributedQueryElevationComponentTest, LeaderFailureAfterFreshStartTest, CreateCollectionCleanupTest, TestLegacyNumericRangeQueryBuilder, LeaderElectionIntegrationTest, TestJsonRequest, TestRandomRequestDistribution, TestClassicSimilarityFactory, TestConfigOverlay, TestRTimerTree, MoveReplicaHDFSTest, TestPhraseSuggestions, TestSQLHandlerNonCloud, TestHdfsCloudBackupRestore]
[junit4] Completed [123/744 (1!)] on J0 in 50.45s, 1 test, 1 failure <<< FAILURES!
[...truncated 49026 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 202 - Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/202/
All tests passed
Build Log:
[...truncated 4708 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:826: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:770: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build.xml:495: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:2262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:409: impossible to resolve dependencies:
resolve failed - see output for details
Total time: 115 minutes 8 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 201 - Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/201/
All tests passed
Build Log:
[...truncated 4723 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:826: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:770: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build.xml:495: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:2262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:409: impossible to resolve dependencies:
resolve failed - see output for details
Total time: 117 minutes 26 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 200 - Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/200/
All tests passed
Build Log:
[...truncated 1364 lines...]
[junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5418177546657407218320.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] #
[junit4] # There is insufficient memory for the Java Runtime Environment to continue.
[junit4] # Native memory allocation (mmap) failed to map 41418752 bytes for committing reserved memory.
[junit4] # An error report file with more information is saved as:
[junit4] # /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/J1/hs_err_pid10311.log
[junit4] <<< JVM J1: EOF ----
[junit4] JVM J1: stderr was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5411283484380853938762.syserr
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000000ed080000, 41418752, 0) failed; error='Cannot allocate memory' (errno=12)
[junit4] <<< JVM J1: EOF ----
[...truncated 288 lines...]
[junit4] ERROR: JVM J1 ended with an exception, command line: /usr/local/asfpackages/java/jdk1.8.0_144/jre/bin/java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=AB360698341A6835 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=europarl.lines.txt.gz -Dtests.luceneMatchVersion=7.2.0 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/tools/junit4/logging.properties -Dtests.nightly=false -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=2 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp -Dcommon.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene -Dclover.db.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/clover/db -Djava.security.policy=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/tools/junit4/tests.policy -Dtests.LUCENE_VERSION=7.2.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.src.home=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x -Djava.security.egd=file:/dev/./urandom -Djunit4.childvm.cwd=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/J1 -Djunit4.childvm.id=1 -Djunit4.childvm.count=3 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Dfile.encoding=ISO-8859-1 -classpath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/codecs/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/test-framework/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/test-framework/lib/junit-4.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/test-framework/lib/randomizedtesting-runner-2.5.3.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/classes/test:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5418151426299979181834.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5412074331131381671030.suites -stdin
[junit4] ERROR: JVM J1 ended with an exception: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5418177546657407218320.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5411283484380853938762.syserr
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] at java.lang.Thread.run(Thread.java:748)
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:826: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:770: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build.xml:50: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1494: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1021: At least one slave process threw an exception, first: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5418177546657407218320.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5411283484380853938762.syserr
Total time: 46 minutes 59 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-Tests-7.x #197 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 199 - Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/199/
1 tests failed.
FAILED: org.apache.solr.cloud.TestLocalFSCloudBackupRestore.test
Error Message:
expected:<COMPLETED> but was:<RUNNING>
Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
at __randomizedtesting.SeedInfo.seed([39042BD03F00148:8BC47D67AD0C6CB0]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:277)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11578 lines...]
[junit4] Suite: org.apache.solr.cloud.TestLocalFSCloudBackupRestore
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/init-core-data-001
[junit4] 2> 0 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 704 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.e.j.u.log Logging initialized @26167ms
[junit4] 2> 801 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 886 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 1035 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001
[junit4] 2> 1153 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1156 INFO (Thread-1) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1156 INFO (Thread-1) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1884 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:35952
[junit4] 2> 2990 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
[junit4] 2> 3627 ERROR (Thread-1) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 5182 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f7083c0440000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 9513 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 9549 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 9930 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@34e710a4{/solr,null,AVAILABLE}
[junit4] 2> 9964 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7db77681{/solr,null,AVAILABLE}
[junit4] 2> 10051 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@39e9b53{HTTP/1.1,[http/1.1]}{127.0.0.1:41201}
[junit4] 2> 10052 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.Server Started @35612ms
[junit4] 2> 10052 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41201}
[junit4] 2> 10055 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5a6ff595{HTTP/1.1,[http/1.1]}{127.0.0.1:58560}
[junit4] 2> 10056 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.Server Started @35616ms
[junit4] 2> 10056 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58560}
[junit4] 2> 10371 ERROR (jetty-launcher-1-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 10371 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 10375 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 10375 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 10376 ERROR (jetty-launcher-1-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 10377 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 10428 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 10428 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 10487 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-31T03:40:11.947Z
[junit4] 2> 10551 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-31T03:40:11.911Z
[junit4] 2> 10667 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
[junit4] 2> 10972 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 11026 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 18546 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35952/solr
[junit4] 2> 18551 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35952/solr
[junit4] 2> 18680 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f7083c0440004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 20262 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 20270 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:41201_solr
[junit4] 2> 20280 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 20303 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.c.Overseer Overseer (id=98921427994148869-127.0.0.1:41201_solr-n_0000000000) starting
[junit4] 2> 20307 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58560_solr
[junit4] 2> 20378 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 20379 INFO (zkCallback-12-thread-1-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 21700 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41201_solr
[junit4] 2> 21757 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 21763 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 27189 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27221 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27222 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27466 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27500 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27500 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27692 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/.
[junit4] 2> 27764 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/.
[junit4] 2> 29970 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 29972 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35952/solr ready
[junit4] 2> 30097 INFO (TEST-TestLocalFSCloudBackupRestore.test-seed#[39042BD03F00148]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 33822 INFO (qtp1422851308-35) [n:127.0.0.1:58560_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&router.name=implicit&version=2&pullReplicas=1&shards=shard1,shard2&property.customKey=customValue&maxShardsPerNode=3&router.field=shard_s&autoAddReplicas=true&name=backuprestore&nrtReplicas=1&action=CREATE&tlogReplicas=1&wt=javabin and sendToOCPQueue=true
[junit4] 2> 34064 INFO (OverseerThreadFactory-12-thread-1-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.CreateCollectionCmd Create collection backuprestore
[junit4] 2> 34070 WARN (OverseerThreadFactory-12-thread-1-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 3 on collection backuprestore is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
[junit4] 2> 34296 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"backuprestore_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:58560/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 34534 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"backuprestore_shard1_replica_t2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:41201/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 34536 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"backuprestore_shard1_replica_p3",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:58560/solr",
[junit4] 2> "type":"PULL",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 34552 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"backuprestore_shard2_replica_n4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:41201/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 34553 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"backuprestore_shard2_replica_t5",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:58560/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 34572 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"backuprestore_shard2_replica_p6",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:41201/solr",
[junit4] 2> "type":"PULL",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 35287 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node11&name=backuprestore_shard2_replica_t5&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 35287 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node10&name=backuprestore_shard2_replica_n4&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 35325 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node9&name=backuprestore_shard1_replica_p3&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 35326 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node7&name=backuprestore_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 35329 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node8&name=backuprestore_shard1_replica_t2&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 35388 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 35390 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 35416 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node12&name=backuprestore_shard2_replica_p6&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 35721 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 35727 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 36801 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 36843 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 38736 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 38756 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 38770 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.IndexSchema [backuprestore_shard2_replica_p6] Schema name=minimal
[junit4] 2> 38776 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 38820 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 38848 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.IndexSchema [backuprestore_shard1_replica_t2] Schema name=minimal
[junit4] 2> 40270 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.IndexSchema [backuprestore_shard1_replica_p3] Schema name=minimal
[junit4] 2> 40279 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.s.IndexSchema [backuprestore_shard2_replica_t5] Schema name=minimal
[junit4] 2> 40284 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.s.IndexSchema [backuprestore_shard2_replica_n4] Schema name=minimal
[junit4] 2> 40337 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.s.IndexSchema [backuprestore_shard1_replica_n1] Schema name=minimal
[junit4] 2> 40625 WARN (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40630 WARN (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieLongField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40655 WARN (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40659 WARN (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40670 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 40671 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_n4' using configuration from collection backuprestore, trusted=true
[junit4] 2> 40671 WARN (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40679 WARN (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40680 WARN (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40782 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 40782 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_p6' using configuration from collection backuprestore, trusted=true
[junit4] 2> 41017 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 41022 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 41022 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 41017 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_t2' using configuration from collection backuprestore, trusted=true
[junit4] 2> 41028 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 41028 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_t5' using configuration from collection backuprestore, trusted=true
[junit4] 2> 41022 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_n1' using configuration from collection backuprestore, trusted=true
[junit4] 2> 41022 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_p3' using configuration from collection backuprestore, trusted=true
[junit4] 2> 41228 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.core.backuprestore.shard1.replica_t2' (registry 'solr.core.backuprestore.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41232 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.core.backuprestore.shard1.replica_p3' (registry 'solr.core.backuprestore.shard1.replica_p3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41232 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.core.backuprestore.shard1.replica_n1' (registry 'solr.core.backuprestore.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41232 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.core.backuprestore.shard2.replica_t5' (registry 'solr.core.backuprestore.shard2.replica_t5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41233 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.core.backuprestore.shard2.replica_p6' (registry 'solr.core.backuprestore.shard2.replica_p6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41253 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41263 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41268 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41268 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41270 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrCore [[backuprestore_shard2_replica_p6] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/backuprestore_shard2_replica_p6], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/./backuprestore_shard2_replica_p6/data/]
[junit4] 2> 41284 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SolrCore [[backuprestore_shard2_replica_t5] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/backuprestore_shard2_replica_t5], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/./backuprestore_shard2_replica_t5/data/]
[junit4] 2> 41299 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrCore [[backuprestore_shard1_replica_p3] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/backuprestore_shard1_replica_p3], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/./backuprestore_shard1_replica_p3/data/]
[junit4] 2> 41363 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrCore [[backuprestore_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/backuprestore_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/./backuprestore_shard1_replica_n1/data/]
[junit4] 2> 41367 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41367 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore [[backuprestore_shard1_replica_t2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/backuprestore_shard1_replica_t2], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/./backuprestore_shard1_replica_t2/data/]
[junit4] 2> 41376 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.core.backuprestore.shard2.replica_n4' (registry 'solr.core.backuprestore.shard2.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41376 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41376 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrCore [[backuprestore_shard2_replica_n4] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/backuprestore_shard2_replica_n4], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/./backuprestore_shard2_replica_n4/data/]
[junit4] 2> 52558 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 52558 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 52668 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 52711 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 52863 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 52863 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 52876 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 53333 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 53392 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 53392 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 53394 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 53394 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 53423 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 53424 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 53437 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 53437 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 53439 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 53439 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 53469 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 53470 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 53513 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@4ec697d8[backuprestore_shard2_replica_n4] main]
[junit4] 2> 53532 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.s.SolrIndexSearcher Opening [Searcher@314e8040[backuprestore_shard2_replica_t5] main]
[junit4] 2> 53532 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@bc28726[backuprestore_shard1_replica_n1] main]
[junit4] 2> 53533 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@3347e5b6[backuprestore_shard1_replica_t2] main]
[junit4] 2> 53559 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.SolrIndexSearcher Opening [Searcher@372436bd[backuprestore_shard2_replica_p6] main]
[junit4] 2> 53624 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.SolrIndexSearcher Opening [Searcher@76bf53d[backuprestore_shard1_replica_p3] main]
[junit4] 2> 53665 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53667 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53708 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53715 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53715 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53731 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53740 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53740 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53755 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53760 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53765 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53775 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53841 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 53878 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 53897 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 53897 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 53848 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 53898 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 57876 WARN (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker-SendThread(127.0.0.1:35952)) [ ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 6712ms for sessionid 0x15f7083c0440008
[junit4] 2> 57877 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f7083c0440008, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 58083 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582742906708426752
[junit4] 2> 58264 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 58266 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 58284 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 59122 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.ZkController backuprestore_shard1_replica_p3 starting background replication from leader
[junit4] 2> 59125 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582742907801042944
[junit4] 2> 59125 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.ZkController backuprestore_shard2_replica_p6 starting background replication from leader
[junit4] 2> 59126 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582742907802091520
[junit4] 2> 59132 WARN (zkCallback-20-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5077fabf name: ZooKeeperConnection Watcher:127.0.0.1:35952/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 59132 WARN (zkCallback-20-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 59216 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 59293 INFO (searcherExecutor-20-thread-1-processing-n:127.0.0.1:58560_solr x:backuprestore_shard1_replica_n1 s:shard1 c:backuprestore r:core_node7) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrCore [backuprestore_shard1_replica_n1] Registered new searcher Searcher@bc28726[backuprestore_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59294 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 59294 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 59295 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:58560/solr/backuprestore_shard1_replica_n1/
[junit4] 2> 59421 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=backuprestore_shard1_replica_n1 url=http://127.0.0.1:58560/solr START replicas=[http://127.0.0.1:41201/solr/backuprestore_shard1_replica_t2/] nUpdates=100
[junit4] 2> 59770 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=backuprestore_shard1_replica_n1 url=http://127.0.0.1:58560/solr DONE. We have no versions. sync failed.
[junit4] 2> 59773 INFO (searcherExecutor-18-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard2_replica_p6 s:shard2 c:backuprestore r:core_node12) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrCore [backuprestore_shard2_replica_p6] Registered new searcher Searcher@372436bd[backuprestore_shard2_replica_p6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59777 INFO (searcherExecutor-21-thread-1-processing-n:127.0.0.1:58560_solr x:backuprestore_shard2_replica_t5 s:shard2 c:backuprestore r:core_node11) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SolrCore [backuprestore_shard2_replica_t5] Registered new searcher Searcher@314e8040[backuprestore_shard2_replica_t5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59779 INFO (searcherExecutor-22-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard1_replica_t2 s:shard1 c:backuprestore r:core_node8) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore [backuprestore_shard1_replica_t2] Registered new searcher Searcher@3347e5b6[backuprestore_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59780 INFO (searcherExecutor-17-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard2_replica_n4 s:shard2 c:backuprestore r:core_node10) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrCore [backuprestore_shard2_replica_n4] Registered new searcher Searcher@4ec697d8[backuprestore_shard2_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59782 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582742908489957376
[junit4] 2> 59231 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 59804 INFO (searcherExecutor-19-thread-1-processing-n:127.0.0.1:58560_solr x:backuprestore_shard1_replica_p3 s:shard1 c:backuprestore r:core_node9) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrCore [backuprestore_shard1_replica_p3] Registered new searcher Searcher@76bf53d[backuprestore_shard1_replica_p3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59845 INFO (qtp1468422210-28) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.S.Request [backuprestore_shard1_replica_t2] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=392
[junit4] 2> 59851 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=9999ms
[junit4] 2> 59882 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 59882 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 59883 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 59883 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 60015 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60016 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60417 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 60417 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 60417 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:58560/solr/backuprestore_shard2_replica_t5/
[junit4] 2> 60419 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.PeerSync PeerSync: core=backuprestore_shard2_replica_t5 url=http://127.0.0.1:58560/solr START replicas=[http://127.0.0.1:41201/solr/backuprestore_shard2_replica_n4/] nUpdates=100
[junit4] 2> 60531 INFO (qtp1468422210-31) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.S.Request [backuprestore_shard2_replica_n4] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 60531 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 60532 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 60532 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 60616 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:58560/solr/backuprestore_shard1_replica_n1/ shard1
[junit4] 2> 60717 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60717 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60731 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.PeerSync PeerSync: core=backuprestore_shard2_replica_t5 url=http://127.0.0.1:58560/solr DONE. We have no versions. sync failed.
[junit4] 2> 60731 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 60731 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 60732 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 60732 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 60736 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ZkController backuprestore_shard2_replica_t5 stopping background replication from leader
[junit4] 2> 60756 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:58560/solr/backuprestore_shard2_replica_t5/ shard2
[junit4] 2> 60757 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node7&name=backuprestore_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin} status=0 QTime=25431
[junit4] 2> 60759 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Replica core_node7 is leader but it's state is down, skipping replication
[junit4] 2> 60860 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60861 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60863 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 60880 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node11&name=backuprestore_shard2_replica_t5&action=CREATE&numShards=2&shard=shard2&wt=javabin} status=0 QTime=25599
[junit4] 2> 60882 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Replica core_node11 is leader but it's state is down, skipping replication
[junit4] 2> 60903 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node10&name=backuprestore_shard2_replica_n4&action=CREATE&numShards=2&shard=shard2&wt=javabin} status=0 QTime=25619
[junit4] 2> 60905 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node9&name=backuprestore_shard1_replica_p3&action=CREATE&numShards=2&shard=shard1&wt=javabin} status=0 QTime=25580
[junit4] 2> 60940 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node12&name=backuprestore_shard2_replica_p6&action=CREATE&numShards=2&shard=shard2&wt=javabin} status=0 QTime=25524
[junit4] 2> 61052 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 61053 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 61432 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.ZkController backuprestore_shard1_replica_t2 starting background replication from leader
[junit4] 2> 61432 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 61449 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 61449 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 61452 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node8&name=backuprestore_shard1_replica_t2&action=CREATE&numShards=2&shard=shard1&wt=javabin} status=0 QTime=26123
[junit4] 2> 61461 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:58560/solr/backuprestore_shard1_replica_n1/
[junit4] 2> 61615 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 61616 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 61628 INFO (qtp1422851308-36) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.S.Request [backuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=150
[junit4] 2> 61636 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 61648 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 61648 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 61648 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 61649 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 61676 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@7af42f8f[backuprestore_shard1_replica_t2] main]
[junit4] 2> 61705 INFO (searcherExecutor-22-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard1_replica_t2 s:shard1 c:backuprestore r:core_node8) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore [backuprestore_shard1_replica_t2] Registered new searcher Searcher@7af42f8f[backuprestore_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 61965 INFO (OverseerCollectionConfigSetProcessor-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 62660 INFO (qtp1422851308-35) [n:127.0.0.1:58560_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 62661 INFO (qtp1422851308-35) [n:127.0.0.1:58560_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&router.name=implicit&version=2&pullReplicas=1&shards=shard1,shard2&property.customKey=customValue&maxShardsPerNode=3&router.field=shard_s&autoAddReplicas=true&name=backuprestore&nrtReplicas=1&action=CREATE&tlogReplicas=1&wt=javabin} status=0 QTime=29127
[junit4] 2> 62883 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication
[junit4] 2> 62884 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:58560/solr/backuprestore_shard1_replica_n1/
[junit4] 2> 62885 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication
[junit4] 2> 62885 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:58560/solr/backuprestore_shard2_replica_t5/
[junit4] 2> 62886 INFO (qtp1422851308-39) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.S.Request [backuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 62892 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 62892 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 62892 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 62892 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 62892 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 62896 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 62896 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2c8c1df2 commitCommandVersion:0
[junit4] 2> 62936 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.S.Request [backuprestore_shard2_replica_t5] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=2
[junit4] 2> 62937 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 62937 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 62937 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 62937 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 62937 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 62938 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 62938 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@70ddacb8 commitCommandVersion:0
[junit4] 2> 62952 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.SolrIndexSearcher Opening [Searcher@582d8e1b[backuprestore_shard1_replica_p3] main]
[junit4] 2> 62968 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.SolrIndexSearcher Opening [Searcher@1dbec7a1[backuprestore_shard2_replica_p6] main]
[junit4] 2> 62969 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 62970 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 63636 INFO (searcherExecutor-18-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard2_replica_p6 s:shard2 c:backuprestore r:core_node12) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrCore [backuprestore_shard2_replica_p6] Registered new searcher Searcher@1dbec7a1[backuprestore_shard2_replica_p6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 63699 INFO (searcherExecutor-19-thread-1-processing-n:127.0.0.1:58560_solr x:backuprestore_shard1_replica_p3 s:shard1 c:backuprestore r:core_node9) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrCore [backuprestore_shard1_replica_p3] Registered new searcher Searcher@582d8e1b[backuprestore_shard1_replica_p3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 64458 INFO (qtp1422851308-32) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.S.Request [backuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 64460 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 64460 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 64460 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 64460 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 64460 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 64480 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@5fd941fa[backuprestore_shard1_replica_t2] main]
[junit4] 2> 64483 INFO (searcherExecutor-22-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard1_replica_t2 s:shard1 c:backuprestore r:core_node8) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore [backuprestore_shard1_replica_t2] Registered new searcher Searcher@5fd941fa[backuprestore_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 65903 INFO (qtp1422851308-32) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.S.Request [backuprestore_shard2_replica_t5] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Slave's generation: 2
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Slave's version: 1509421264457
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@70ddacb8 commitCommandVersion:0
[junit4] 2> 65928 INFO (qtp1422851308-39) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.S.Request [backuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Slave's generation: 2
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Slave's version: 1509421264415
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2c8c1df2 commitCommandVersion:0
[junit4] 2> 65948 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.SolrIndexSearcher Opening [Searcher@3aa70f9b[backuprestore_shard2_replica_p6] main]
[junit4] 2> 65948 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 65951 INFO (searcherExecutor-18-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard2_replica_p6 s:shard2 c:backuprestore r:core_node12) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrCore [backuprestore_shard2_replica_p6] Registered new searcher Searcher@3aa70f9b[backuprestore_shard2_replica_p6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 65953 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.SolrIndexSearcher Opening [Searcher@8d8fcd8[backuprestore_shard1_replica_p3] main]
[junit4] 2> 65953 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 65958 INFO (searcherExecutor-19-thread-1-processing-n:127.0.0.1:58560_solr x:backuprestore_shard1_replica_p3 s:shard1 c:backuprestore r:core_node9) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrCore [backuprestore_shard1_replica_p3] Registered new searcher Searcher@8d8fcd8[backuprestore_shard1_replica_p3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 67456 INFO (qtp1422851308-32) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.S.Request [backuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 67457 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 67457 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 67457 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 67457 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 67457 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 67471 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4ed43433[backuprestore_shard1_replica_t2] main]
[junit4] 2> 67475 INFO (searcherExecutor-22-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard1_replica_t2 s:shard1 c:backuprestore r:core_node8) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore [backuprestore_shard1_replica_t2] Registered new searcher Searcher@4ed43433[backuprestore_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 67500 INFO (qtp1468422210-31) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [backuprestore_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:58560/solr/backuprestore_shard1_replica_n1/&wt=javabin&version=2}{add=[0 (1582742912532217856), 1 (1582742915467182080), 4 (1582742915491299328), 6 (1582742915497590784), 7 (1582742915514368000), 8 (1582742915646488576), 10 (1582742915649634304), 13 (1582742915684237312), 14 (1582742915689480192), 17 (1582742915
[...truncated too long message...]
ave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptor-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-admin-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-authn-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/byte-buddy-1.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/commons-math3-3.6.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/mockito-core-2.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/objenesis-2.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/analysis/icu/lucene-analyzers-icu-7.2.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/contrib/analysis-extras/lib/icu4j-59.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20171031_033935_3816929525940187101917.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20171031_033935_3814221955659416896517.suites -stdin
[junit4] ERROR: JVM J2 ended with an exception: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20171031_033935_3814842593785503032539.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20171031_033935_3814372273523413635019.syserr
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] at java.lang.Thread.run(Thread.java:748)
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:826: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:770: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build.xml:262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/common-build.xml:556: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1494: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1021: At least one slave process threw an exception, first: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J0-20171031_033935_3321840776022627920860.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J0-20171031_033935_3326211161918442131645.syserr
Total time: 235 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-Tests-7.x #197 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 198 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/198/
No tests ran.
Build Log:
[...truncated 7005 lines...]
ERROR: command execution failed.
ERROR: Step ‘Archive the artifacts’ failed: no workspace for Lucene-Solr-Tests-7.x #198
ERROR: Step ‘Publish JUnit test result report’ failed: no workspace for Lucene-Solr-Tests-7.x #198
ERROR: lucene is offline; cannot locate JDK 1.8 (latest)
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
ERROR: lucene is offline; cannot locate JDK 1.8 (latest)
ERROR: lucene is offline; cannot locate JDK 1.8 (latest)
[JENKINS] Lucene-Solr-Tests-7.x - Build # 197 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/197/
10 tests failed.
FAILED: org.apache.solr.cloud.ShardRoutingTest.test
Error Message:
Error from server at http://127.0.0.1:34143: At least one of the node(s) specified are not currently active, no action taken.
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34143: At least one of the node(s) specified are not currently active, no action taken.
at __randomizedtesting.SeedInfo.seed([858C689B840B3A43:DD857412AF757BB]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1096)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:875)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:418)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ShardRoutingTest
Error Message:
198 threads leaked from SUITE scope at org.apache.solr.cloud.ShardRoutingTest: 1) Thread[id=4679, name=qtp383150318-4679, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=4599, name=Scheduler-351568579, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=4534, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 4) Thread[id=4607, name=qtp331754181-4607-acceptor-0@17c44e2f-ServerConnector@67347bd3{HTTP/1.1,[http/1.1]}{127.0.0.1:59846}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=4588, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=4730, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 7) Thread[id=4675, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=4697, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-EventThread, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 9) Thread[id=4504, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 10) Thread[id=4493, name=qtp2059076216-4493, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=4584, name=org.eclipse.jetty.server.session.HashSessionManager@255a5ef9Timer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=4764, name=zkCallback-809-thread-4, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=4706, name=updateExecutor-832-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=4523, name=qtp1505717836-4523, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=4748, name=zkCallback-797-thread-4, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=4543, name=searcherExecutor-1935-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=4705, name=searcherExecutor-1990-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=4530, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=4650, name=org.eclipse.jetty.server.session.HashSessionManager@1ff96782Timer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=4754, name=zkCallback-842-thread-2, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=4671, name=searcherExecutor-1979-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=4711, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=4610, name=qtp331754181-4610, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=4756, name=zkCallback-842-thread-4, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=4618, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=4554, name=org.eclipse.jetty.server.session.HashSessionManager@e00fddcTimer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=4647, name=qtp1351042721-4647, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=4769, name=zkCallback-821-thread-5, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=4721, name=org.eclipse.jetty.server.session.HashSessionManager@6510dcd6Timer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=4613, name=org.eclipse.jetty.server.session.HashSessionManager@635d8de3Timer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=4500, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=4681, name=qtp383150318-4681-acceptor-0@603f0db1-ServerConnector@79281ec4{HTTP/1.1,[http/1.1]}{127.0.0.1:50183}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=4762, name=zkCallback-809-thread-2, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=4668, name=watches-829-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=4725, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=4578, name=qtp1094907735-4578-acceptor-0@1d2723af-ServerConnector@32599859{HTTP/1.1,[http/1.1]}{127.0.0.1:37045}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 37) Thread[id=4563, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 38) Thread[id=4595, name=zkCallback-815-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=4682, name=qtp383150318-4682, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 40) Thread[id=4715, name=qtp1837148975-4715, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 41) Thread[id=4698, name=zkCallback-835-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 42) Thread[id=4719, name=qtp1837148975-4719-acceptor-0@38f54143-ServerConnector@2b2cf0f1{HTTP/1.1,[http/1.1]}{127.0.0.1:40817}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 43) Thread[id=4524, name=qtp1505717836-4524, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 44) Thread[id=4511, name=Scheduler-1049167543, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 45) Thread[id=4535, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-EventThread, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 46) Thread[id=4536, name=zkCallback-803-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 47) Thread[id=4773, name=zkCallback-828-thread-5, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 48) Thread[id=4684, name=qtp383150318-4684, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 49) Thread[id=4577, name=qtp1094907735-4577, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 50) Thread[id=4643, name=qtp1351042721-4643, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 51) Thread[id=4488, name=qtp2059076216-4488, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 52) Thread[id=4759, name=zkCallback-815-thread-3, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 53) Thread[id=4519, name=qtp1505717836-4519-acceptor-0@75539a1e-ServerConnector@321d6f5c{HTTP/1.1,[http/1.1]}{127.0.0.1:38072}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 54) Thread[id=4624, name=zkCallback-821-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 55) Thread[id=4776, name=zkCallback-835-thread-4, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 56) Thread[id=4752, name=zkCallback-803-thread-4, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 57) Thread[id=4770, name=zkCallback-828-thread-2, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 58) Thread[id=4579, name=qtp1094907735-4579, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 59) Thread[id=4645, name=qtp1351042721-4645, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 60) Thread[id=4529, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 61) Thread[id=4546, name=qtp21053055-4546, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 62) Thread[id=4696, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 63) Thread[id=4648, name=qtp1351042721-4648, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 64) Thread[id=4766, name=zkCallback-821-thread-2, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 65) Thread[id=4665, name=coreContainerWorkExecutor-1974-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 66) Thread[id=4573, name=searcherExecutor-1946-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 67) Thread[id=4514, name=searcherExecutor-1924-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 68) Thread[id=4718, name=qtp1837148975-4718, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 69) Thread[id=4751, name=zkCallback-803-thread-3, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 70) Thread[id=4608, name=qtp331754181-4608, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 71) Thread[id=4644, name=qtp1351042721-4644-acceptor-0@7ce398e3-ServerConnector@3bd4dc11{HTTP/1.1,[http/1.1]}{127.0.0.1:40270}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 72) Thread[id=4699, name=Thread-1013, state=WAITING, group=TGRP-ShardRoutingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 73) Thread[id=4553, name=qtp21053055-4553, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 74) Thread[id=4717, name=qtp1837148975-4717, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 75) Thread[id=4749, name=zkCallback-797-thread-5, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 76) Thread[id=4753, name=zkCallback-803-thread-5, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 77) Thread[id=4537, name=Thread-968, state=WAITING, group=TGRP-ShardRoutingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 78) Thread[id=4758, name=zkCallback-815-thread-2, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 79) Thread[id=4767, name=zkCallback-821-thread-3, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 80) Thread[id=4558, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 81) Thread[id=4771, name=zkCallback-828-thread-3, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 82) Thread[id=4583, name=qtp1094907735-4583, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 83) Thread[id=4507, name=Thread-959, state=WAITING, group=TGRP-ShardRoutingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 84) Thread[id=4489, name=qtp2059076216-4489-acceptor-0@283458db-ServerConnector@7a134e2f{HTTP/1.1,[http/1.1]}{127.0.0.1:36211}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 85) Thread[id=4633, name=searcherExecutor-1968-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 86) Thread[id=4700, name=coreContainerWorkExecutor-1985-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 87) Thread[id=4622, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 88) Thread[id=4506, name=zkCallback-797-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 89) Thread[id=4520, name=qtp1505717836-4520, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 90) Thread[id=4525, name=org.eclipse.jetty.server.session.HashSessionManager@38d81390Timer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 91) Thread[id=4716, name=qtp1837148975-4716, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 92) Thread[id=4550, name=qtp21053055-4550, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 93) Thread[id=4740, name=OverseerCollectionConfigSetProcessor-98916296398602249-127.0.0.1:36211_-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 94) Thread[id=4663, name=zkCallback-828-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 95) Thread[id=4691, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 96) Thread[id=4547, name=qtp21053055-4547, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 97) Thread[id=4636, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 98) Thread[id=4487, name=qtp2059076216-4487, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 99) Thread[id=4491, name=qtp2059076216-4491, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 100) Thread[id=4739, name=OverseerStateUpdate-98916296398602249-127.0.0.1:36211_-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:202) at java.lang.Thread.run(Thread.java:748) 101) Thread[id=4551, name=qtp21053055-4551, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 102) Thread[id=4726, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 103) Thread[id=4657, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 104) Thread[id=4649, name=qtp1351042721-4649, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 105) Thread[id=4714, name=qtp1837148975-4714, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 106) Thread[id=4692, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 107) Thread[id=4755, name=zkCallback-842-thread-3, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 108) Thread[id=4494, name=qtp2059076216-4494, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 109) Thread[id=4617, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 110) Thread[id=4778, name=ScheduledTrigger-2004-thread-3, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 111) Thread[id=4559, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 112) Thread[id=4549, name=qtp21053055-4549, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 113) Thread[id=4548, name=qtp21053055-4548-acceptor-0@78d4bb24-ServerConnector@3109465d{HTTP/1.1,[http/1.1]}{127.0.0.1:34143}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 114) Thread[id=4545, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 115) Thread[id=4512, name=watches-798-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 116) Thread[id=4566, name=Thread-977, state=WAITING, group=TGRP-ShardRoutingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 117) Thread[id=4777, name=zkCallback-835-thread-5, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurren
[...truncated too long message...]
rent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 18) Thread[id=6926, name=TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45890), state=TIMED_WAITING, group=TGRP-ShardSplitTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > 19) Thread[id=6900, name=Thread-1417, state=WAITING, group=TGRP-ShardSplitTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678)
[junit4] > 20) Thread[id=6881, name=qtp1732939943-6881, state=RUNNABLE, group=TGRP-ShardSplitTest]
[junit4] > at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 21) Thread[id=6871, name=Thread-1408, state=WAITING, group=TGRP-ShardSplitTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678)
[junit4] > 22) Thread[id=6943, name=qtp199264780-6943, state=RUNNABLE, group=TGRP-ShardSplitTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
[junit4] > at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
[junit4] > at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
[junit4] > at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 23) Thread[id=6935, name=searcherExecutor-3171-thread-1, state=WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 24) Thread[id=6874, name=Scheduler-821479584, state=WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 25) Thread[id=6884, name=qtp1732939943-6884, state=RUNNABLE, group=TGRP-ShardSplitTest]
[junit4] > at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 26) Thread[id=6915, name=qtp551973238-6915, state=RUNNABLE, group=TGRP-ShardSplitTest]
[junit4] > at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([858C689B840B3A43]:0)
[junit4] Completed [688/745 (5!)] on J0 in 578.91s, 5 tests, 1 failure, 3 errors <<< FAILURES!
[...truncated 47426 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 196 - Still unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/196/
1 tests failed.
FAILED: org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader
Error Message:
Could not load collection from ZK: forceleader_test_collection
Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: forceleader_test_collection
at __randomizedtesting.SeedInfo.seed([8BC74AD0A95B1D43:6D507E1090D9E422]:0)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
at org.apache.solr.common.cloud.ZkStateReader.getLeader(ZkStateReader.java:787)
at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:809)
at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:547)
at org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader(ForceLeaderTest.java:143)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/forceleader_test_collection/state.json
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
... 47 more
Build Log:
[...truncated 13453 lines...]
[junit4] Suite: org.apache.solr.cloud.ForceLeaderTest
[junit4] 2> 5056756 INFO (SUITE-ForceLeaderTest-seed#[8BC74AD0A95B1D43]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/init-core-data-001
[junit4] 2> 5056757 WARN (SUITE-ForceLeaderTest-seed#[8BC74AD0A95B1D43]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=14 numCloses=14
[junit4] 2> 5056757 INFO (SUITE-ForceLeaderTest-seed#[8BC74AD0A95B1D43]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 5056791 INFO (SUITE-ForceLeaderTest-seed#[8BC74AD0A95B1D43]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 5056791 INFO (SUITE-ForceLeaderTest-seed#[8BC74AD0A95B1D43]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /_/e
[junit4] 2> 5056848 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 5056848 INFO (Thread-2793) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 5056848 INFO (Thread-2793) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 5056883 ERROR (Thread-2793) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 5056956 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkTestServer start zk server on port:39184
[junit4] 2> 5057031 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 5057032 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 5057045 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 5057046 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 5057047 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 5057047 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 5057048 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 5057048 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 5057049 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 5057083 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 5057084 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 5057103 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 5058141 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 5058284 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@63404405{/_/e,null,AVAILABLE}
[junit4] 2> 5058284 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@705906fa{HTTP/1.1,[http/1.1]}{127.0.0.1:37146}
[junit4] 2> 5058284 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server Started @5075560ms
[junit4] 2> 5058285 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/tempDir-001/control/data, replicaType=NRT, hostContext=/_/e, hostPort=43207, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/control-001/cores}
[junit4] 2> 5058285 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5058285 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 5058285 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5058285 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5058285 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-28T10:57:06.126Z
[junit4] 2> 5058305 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5058305 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/control-001/solr.xml
[junit4] 2> 5058378 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 5058378 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 5058379 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5058518 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39184/solr
[junit4] 2> 5059052 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5059065 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43207__%2Fe
[junit4] 2> 5059066 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.Overseer Overseer (id=98906159494201348-127.0.0.1:43207__%2Fe-n_0000000000) starting
[junit4] 2> 5059140 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43207__%2Fe
[junit4] 2> 5059175 INFO (zkCallback-1780-thread-1-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5059726 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5059872 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5059873 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5059907 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/control-001/cores
[junit4] 2> 5060110 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5060111 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39184/solr ready
[junit4] 2> 5060111 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=45701,localport=43207], receiveBufferSize:531000
[junit4] 2> 5060111 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37146,localport=58176], receiveBufferSize=530904
[junit4] 2> 5060165 INFO (qtp1430705358-12240) [n:127.0.0.1:43207__%2Fe ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:43207__%252Fe&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5060210 INFO (OverseerThreadFactory-5958-thread-1-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 5060353 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=45704,localport=43207], receiveBufferSize:531000
[junit4] 2> 5060364 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37146,localport=58180], receiveBufferSize=530904
[junit4] 2> 5060373 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 5060373 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 5060697 INFO (zkCallback-1780-thread-1-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 5061673 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 5061766 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 5062892 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 5063355 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 5063356 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5063356 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5063356 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 5063407 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39538988765470406]
[junit4] 2> 5063429 WARN (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 5064014 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5064014 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5064015 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5064015 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5064016 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1723384108, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 5064065 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4817dc57[control_collection_shard1_replica_n1] main]
[junit4] 2> 5064066 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 5064067 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 5064067 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5064068 INFO (searcherExecutor-5961-thread-1-processing-n:127.0.0.1:43207__%2Fe x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@4817dc57[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5064069 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582498559463260160
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:43207/_/e/control_collection_shard1_replica_n1/
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:43207/_/e/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 5064140 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:43207/_/e/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 5064224 INFO (zkCallback-1780-thread-2-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 5064248 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 5064249 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3876
[junit4] 2> 5064306 INFO (qtp1430705358-12240) [n:127.0.0.1:43207__%2Fe ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 5064448 INFO (zkCallback-1780-thread-1-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 5065306 INFO (qtp1430705358-12240) [n:127.0.0.1:43207__%2Fe ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:43207__%252Fe&wt=javabin&version=2} status=0 QTime=5141
[junit4] 2> 5065490 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5065491 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39184/solr ready
[junit4] 2> 5065491 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 5065491 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=45744,localport=43207], receiveBufferSize:531000
[junit4] 2> 5065492 INFO (qtp1430705358-12241) [n:127.0.0.1:43207__%2Fe ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5065513 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37146,localport=58220], receiveBufferSize=530904
[junit4] 2> 5065526 INFO (OverseerCollectionConfigSetProcessor-98906159494201348-127.0.0.1:43207__%2Fe-n_0000000000) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5065526 INFO (OverseerThreadFactory-5958-thread-2-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 5065527 WARN (OverseerThreadFactory-5958-thread-2-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 5065783 INFO (qtp1430705358-12241) [n:127.0.0.1:43207__%2Fe ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 5065783 INFO (qtp1430705358-12241) [n:127.0.0.1:43207__%2Fe ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=291
[junit4] 2> 5067243 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001 of type TLOG
[junit4] 2> 5067243 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 5067396 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@25b7b6f9{/_/e,null,AVAILABLE}
[junit4] 2> 5067396 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@109cddf1{HTTP/1.1,[http/1.1]}{127.0.0.1:36794}
[junit4] 2> 5067396 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server Started @5084672ms
[junit4] 2> 5067396 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/_/e, hostPort=42162, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001/cores}
[junit4] 2> 5067396 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5067396 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 5067397 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5067397 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5067397 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-28T10:57:15.238Z
[junit4] 2> 5067543 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5067544 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001/solr.xml
[junit4] 2> 5067544 INFO (OverseerCollectionConfigSetProcessor-98906159494201348-127.0.0.1:43207__%2Fe-n_0000000000) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5067599 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 5067599 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 5067615 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5067637 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39184/solr
[junit4] 2> 5067765 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5067766 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5067800 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42162__%2Fe
[junit4] 2> 5067801 INFO (zkCallback-1780-thread-2-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5067920 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5067930 INFO (zkCallback-1788-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5069101 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5069192 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5069192 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5069226 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001/cores
[junit4] 2> 5069393 INFO (SocketProxy-Acceptor-42162) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=41154,localport=42162], receiveBufferSize:531000
[junit4] 2> 5069461 INFO (SocketProxy-Acceptor-42162) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=36794,localport=34552], receiveBufferSize=530904
[junit4] 2> 5069464 INFO (qtp1028606452-12294) [n:127.0.0.1:42162__%2Fe ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:42162__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5069520 INFO (OverseerThreadFactory-5958-thread-3-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:42162__%2Fe for creating new replica
[junit4] 2> 5069539 INFO (SocketProxy-Acceptor-42162) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=41158,localport=42162], receiveBufferSize:531000
[junit4] 2> 5069588 INFO (SocketProxy-Acceptor-42162) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=36794,localport=34556], receiveBufferSize=530904
[junit4] 2> 5069588 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 5069588 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 5069809 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 5069919 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema [collection1_shard2_replica_t41] Schema name=test
[junit4] 2> 5070974 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 5071066 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t41' using configuration from collection collection1, trusted=true
[junit4] 2> 5071067 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t41' (registry 'solr.core.collection1.shard2.replica_t41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5071067 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5071067 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [[collection1_shard2_replica_t41] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001/cores/collection1_shard2_replica_t41], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001/cores/collection1_shard2_replica_t41/data/]
[junit4] 2> 5071102 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39538988765470406]
[junit4] 2> 5071175 WARN (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 5071539 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5071539 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5071541 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5071541 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5071561 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1723384108, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 5071561 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.SolrIndexSearcher Opening [Searcher@74c1f867[collection1_shard2_replica_t41] main]
[junit4] 2> 5071576 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 5071577 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 5071577 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5071578 INFO (searcherExecutor-5972-thread-1-processing-n:127.0.0.1:42162__%2Fe x:collection1_shard2_replica_t41 s:shard2 c:collection1) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [collection1_shard2_replica_t41] Registered new searcher Searcher@74c1f867[collection1_shard2_replica_t41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5071578 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582498567337017344
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:42162/_/e/collection1_shard2_replica_t41/
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy http://127.0.0.1:42162/_/e/collection1_shard2_replica_t41/ has no replicas
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 5071665 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController collection1_shard2_replica_t41 stopping background replication from leader
[junit4] 2> 5071685 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5071685 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:42162/_/e/collection1_shard2_replica_t41/ shard2
[junit4] 2> 5071686 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 5071686 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2098
[junit4] 2> 5071688 INFO (qtp1028606452-12294) [n:127.0.0.1:42162__%2Fe ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:42162__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=2223
[junit4] 2> 5071958 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5073318 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001 of type TLOG
[junit4] 2> 5073319 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 5073427 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7c2fdca1{/_/e,null,AVAILABLE}
[junit4] 2> 5073427 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5543f653{HTTP/1.1,[http/1.1]}{127.0.0.1:33348}
[junit4] 2> 5073427 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server Started @5090703ms
[junit4] 2> 5073427 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/_/e, hostPort=41891, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001/cores}
[junit4] 2> 5073428 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5073465 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 5073465 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5073465 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5073465 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-28T10:57:21.306Z
[junit4] 2> 5073502 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5073503 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001/solr.xml
[junit4] 2> 5073516 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f62a0c6c0000a, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 5073518 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 5073518 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 5073520 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5073553 INFO (OverseerCollectionConfigSetProcessor-98906159494201348-127.0.0.1:43207__%2Fe-n_0000000000) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5073569 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39184/solr
[junit4] 2> 5073755 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 5073772 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5073774 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41891__%2Fe
[junit4] 2> 5073774 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5073886 INFO (zkCallback-1788-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5073959 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5073882 INFO (zkCallback-1780-thread-2-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5075013 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5075124 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5075125 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5075126 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001/cores
[junit4] 2> 5075526 INFO (SocketProxy-Acceptor-41891) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=38412,localport=41891], receiveBufferSize:531000
[junit4] 2> 5075582 INFO (SocketProxy-Acceptor-41891) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=33348,localport=47020], receiveBufferSize=530904
[junit4] 2> 5075603 INFO (qtp994154853-12329) [n:127.0.0.1:41891__%2Fe ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41891__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5075638 INFO (OverseerThreadFactory-5958-thread-4-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:41891__%2Fe for creating new replica
[junit4] 2> 5075692 INFO (SocketProxy-Acceptor-41891) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=38416,localport=41891], receiveBufferSize:531000
[junit4] 2> 5075693 INFO (SocketProxy-Acceptor-41891) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=33348,localport=47024], receiveBufferSize=530904
[junit4] 2> 5075729 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 5075730 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 5075746 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 5076118 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 5076209 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema [collection1_shard1_replica_t43] Schema name=test
[junit4] 2> 5077525 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 5077675 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t43' using configuration from collection collection1, trusted=true
[junit4] 2> 5077676 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t43' (registry 'solr.core.collection1.shard1.replica_t43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5077676 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5077676 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [[collection1_shard1_replica_t43] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001/cores/collection1_shard1_replica_t43], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001/cores/collection1_shard1_replica_t43/data/]
[junit4] 2> 5077691 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39538988765470406]
[junit4] 2> 5077729 WARN (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 5078419 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5078419 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5078420 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5078420 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5078421 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1723384108, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 5078422 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.SolrIndexSearcher Opening [Searcher@54678c50[collection1_shard1_replica_t43] main]
[junit4] 2> 5078439 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 5078439 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 5078440 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5078440 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582498574532345856
[junit4] 2> 5078442 INFO (searcherExecutor-5983-thread-1-processing-n:127.0.0.1:41891__%2Fe x:collection1_shard1_replica_t43 s:shard1 c:collection1) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [collection1_shard1_replica_t43] Registered new searcher Searcher@54678c50[collection1_shard1_replica_t43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:41891/_/e/collection1_shard1_replica_t43/
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy http://127.0.0.1:41891/_/e/collection1_shard1_replica_t43/ has no replicas
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController collection1_shard1_replica_t43 stopping background replication from leader
[junit4] 2> 5078471 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:41891/_/e/collection1_shard1_replica_t43/ shard1
[junit4] 2> 5078493 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 5078499 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 5078528 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 5078530 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2801
[junit4] 2> 5078593 INFO (qtp994154853-12329) [n:127.0.0.1:41891__%2Fe ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:41891__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=2990
[junit4] 2> 5078639 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 5078646 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 5079651 INFO (OverseerCollectionConfigSetProcessor-98906159494201348-127.0.0.1:43207__%2Fe-n_0000000000) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5080691 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001 of type TLOG
[junit4] 2> 5080708 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 5080952 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4a46eab2{/_/e,null,AVAILABLE}
[junit4] 2> 5080953 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6b991fb7{HTTP/1.1,[http/1.1]}{127.0.0.1:39542}
[junit4] 2> 5080953 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server Started @5098228ms
[junit4] 2> 5080953 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/tempDir-001/jetty3, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/_/e, hostPort=41221, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001/cores}
[junit4] 2> 5080953 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5080962 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 5080962 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5080962 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5080963 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-28T10:57:28.804Z
[junit4] 2> 5081054 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5081055 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001/solr.xml
[junit4] 2> 5081074 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 5081074 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 5081145 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5081165 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39184/solr
[junit4] 2> 5081367 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 5081384 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5081385 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41221__%2Fe
[junit4] 2> 5081400 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5081496 INFO (zkCallback-1805-thread-1-processing-n:127.0.0.1:41221__%2Fe) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5081461 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5081461 INFO (zkCallback-1780-thread-1-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5081461 INFO (zkCallback-1788-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5081951 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5082100 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5082133 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5082135 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001/cores
[junit4] 2> 5082537 INFO (SocketProxy-Acceptor-41221) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=53178,localport=41221], receiveBufferSize:531000
[junit4] 2> 5082556 INFO (SocketProxy-Acceptor-41221) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=39542,localport=40134], receiveBufferSize=530904
[junit4] 2> 5082596 INFO (qtp1223044078-12363) [n:127.0.0.1:41221__%2Fe ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41221__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5082704 INFO (OverseerThreadFactory-5958-thread-5-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:41221__%2Fe for creating new replica
[junit4] 2> 5082705 INFO (SocketProxy-Acceptor-41221) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=53182,localport=41221], receiveBufferSize:531000
[junit4] 2> 5082772 INFO (SocketProxy-Acceptor-41221) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=39542,localport=40138], receiveBufferSize=530904
[junit4] 2> 5082772 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 5082773 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 5082794 INFO (zkCallback-1805-thread-1-processing-n:127.0.0.1:41221__%2Fe) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5082794 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5082794 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5083918 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 5084193 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.s.IndexSchema [collection1_shard2_replica_t45] Schema name=test
[junit4] 2> 5085348 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 5085438 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t45' using configuration from collection collection1, trusted=true
[junit4] 2> 5085439 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t45' (registry 'solr.core.collection1.shard2.replica_t45') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5085439 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5085439 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.SolrCore [[collection1_shard2_replica_t45] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001/cores/collection1_shard2_replica_t45], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001/cores/collection1_shard2_replica_t45/data/]
[junit4] 2> 5085494 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39538988765470406]
[junit4] 2> 5085511 WARN (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 5086095 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5086095 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5086096 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5086096 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5086097 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1723384108, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 5086098 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.s.SolrIndexSearcher Opening [Searcher@234ce6ec[collection1_shard2_replica_t45] main]
[junit4] 2> 5086132 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 5086132 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 5086133 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5086134 INFO (searcherExecutor-5994-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.SolrCore [collection1_shard2_replica_t45] Registered new searcher Searcher@234ce6ec[collection1_shard2_replica_t45] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5086134 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582498582600089600
[junit4] 2> 5086186 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.ZkController Core needs to recover:collection1_shard2_replica_t45
[junit4] 2> 5086187 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=3414
[junit4] 2> 5086188 INFO (updateExecutor-1802-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 5086201 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 5086278 INFO (qtp1223044078-12363) [n:127.0.0.1:41221__%2Fe ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:41221__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=3682
[junit4] 2> 5086331 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 5086331 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.ZkController collection1_shard2_replica_t45 stopping background replication from leader
[junit4] 2> 5086331 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard2_replica_t45]
[junit4] 2> 5086331 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 5086331 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard2_replica_t45] as recovering, leader is [http://127.0.0.1:42162/_/e/collection1_shard2_replica_t41/] and I am [http://127.0.0.1:41221/_/e/collection1_shard2_replica_t45/]
[junit4] 2> 5086332 INFO (zkCallback-1805-thread-1-processing-n:127.0.0.1:41221__%2Fe) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5086335 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5086335 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5086344 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testLastPublishedStateIsActive
[junit4] 2> 5086345 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:4216
[...truncated too long message...]
> 5212092 INFO (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40794 40794
[junit4] 2> 5212367 INFO (Thread-2834) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40794 40794
[junit4] 2> 5212387 WARN (Thread-2834) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/aliases.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 4 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/clusterprops.json
[junit4] 2> 7 /solr/clusterstate.json
[junit4] 2> 3 /solr/collections/forceleader_test_collection/state.json
[junit4] 2> 3 /solr/collections/collection1/state.json
[junit4] 2> 3 /solr/autoscaling.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/live_nodes
[junit4] 2> 7 /solr/collections
[junit4] 2> 3 /solr/overseer/collection-queue-work
[junit4] 2> 3 /solr/overseer/queue-work
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2>
[junit4] 2> 5212387 WARN (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SocketProxy Closing 16 connections to: http://127.0.0.1:41867/_/e, target: http://127.0.0.1:34584/_/e
[junit4] 2> 5212400 WARN (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SocketProxy Closing 0 connections to: http://127.0.0.1:36363/_/e, target: http://127.0.0.1:43221/_/e
[junit4] 2> 5212400 WARN (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SocketProxy Closing 4 connections to: http://127.0.0.1:43097/_/e, target: http://127.0.0.1:40800/_/e
[junit4] 2> 5212400 WARN (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SocketProxy Closing 7 connections to: http://127.0.0.1:41330/_/e, target: http://127.0.0.1:43778/_/e
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ForceLeaderTest -Dtests.method=testReplicasInLIRNoLeader -Dtests.seed=8BC74AD0A95B1D43 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=tr-TR -Dtests.timezone=Africa/Ouagadougou -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 64.6s J1 | ForceLeaderTest.testReplicasInLIRNoLeader <<<
[junit4] > Throwable #1: org.apache.solr.common.SolrException: Could not load collection from ZK: forceleader_test_collection
[junit4] > at __randomizedtesting.SeedInfo.seed([8BC74AD0A95B1D43:6D507E1090D9E422]:0)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
[junit4] > at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.getLeader(ZkStateReader.java:787)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:809)
[junit4] > at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:547)
[junit4] > at org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader(ForceLeaderTest.java:143)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/forceleader_test_collection/state.json
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] > at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
[junit4] > ... 47 more
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001
[junit4] 2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=7901, maxDocsPerChunk=2, blockSize=5), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=7901, blockSize=5)), sim=RandomSimilarity(queryNorm=true): {}, locale=tr-TR, timezone=Africa/Ouagadougou
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=274698120,total=522715136
[junit4] 2> NOTE: All tests run in this JVM: [OverseerCollectionConfigSetProcessorTest, TestLazyCores, TestCloudNestedDocsSort, DocValuesMultiTest, DistributedTermsComponentTest, TestOverriddenPrefixQueryForCustomFieldType, TestMaxTokenLenTokenizer, TestExactStatsCache, OpenExchangeRatesOrgProviderTest, HdfsUnloadDistributedZkTest, SolrJmxReporterTest, TestComplexPhraseQParserPlugin, SolrGangliaReporterTest, DirectUpdateHandlerTest, TestZkAclsWithHadoopAuth, CdcrBootstrapTest, TestDefaultStatsCache, DistributedDebugComponentTest, TestSolr4Spatial, TestConfigSetsAPIZkFailure, DistributedFacetPivotLargeTest, TestCloudPivotFacet, TestRTGBase, TestDynamicFieldResource, ShardRoutingTest, CoreAdminRequestStatusTest, TestPseudoReturnFields, TestSolrQueryParser, TestSolrCoreSnapshots, SolrRequestParserTest, TestMacros, HighlighterConfigTest, MBeansHandlerTest, TestScoreJoinQPScore, CdcrRequestHandlerTest, SolrIndexSplitterTest, ConfigureRecoveryStrategyTest, TestManagedResource, TestBackupRepositoryFactory, NodeAddedTriggerTest, OverseerStatusTest, SimpleMLTQParserTest, V2StandaloneTest, TestMiniSolrCloudClusterSSL, TestDistributedStatsComponentCardinality, TestFieldTypeCollectionResource, TestAtomicUpdateErrorCases, PreAnalyzedFieldTest, TestDocumentBuilder, TestFastWriter, AssignTest, TestSegmentSorting, HighlighterMaxOffsetTest, BlobRepositoryCloudTest, TestSerializedLuceneMatchVersion, TestAnalyzedSuggestions, LukeRequestHandlerTest, TestPushWriter, TestManagedSchema, ShowFileRequestHandlerTest, TestSkipOverseerOperations, TestSearcherReuse, TestLegacyTerms, DeleteLastCustomShardedReplicaTest, DateRangeFieldTest, TestFieldCacheWithThreads, HttpPartitionTest, ShardRoutingCustomTest, TestInfoStreamLogging, TestCustomStream, TestRequestForwarding, TestBulkSchemaAPI, SolrIndexConfigTest, ConcurrentDeleteAndCreateCollectionTest, SampleTest, TestCollectionAPIs, HdfsSyncSliceTest, TestPointFields, TestCollationField, TestRangeQuery, TestPivotHelperCode, TestFilteredDocIdSet, SuggesterTSTTest, TestFaceting, TestHighFrequencyDictionaryFactory, TestBinaryResponseWriter, DistribDocExpirationUpdateProcessorTest, UninvertDocValuesMergePolicyTest, TestNumericRangeQuery64, TemplateUpdateProcessorTest, TestReRankQParserPlugin, SpatialRPTFieldTypeTest, TestSortByMinMaxFunction, TestStressRecovery, HLLSerializationTest, TestDocSet, CircularListTest, BlockJoinFacetRandomTest, TestStressReorder, TestDocTermOrds, TestWriterPerf, SpatialHeatmapFacetsTest, ConvertedLegacyTest, ConjunctionSolrSpellCheckerTest, SOLR749Test, TestExceedMaxTermLength, DistributedFacetPivotSmallTest, OpenCloseCoreStressTest, DeleteShardTest, TestQueryWrapperFilter, SharedFSAutoReplicaFailoverTest, HdfsBasicDistributedZk2Test, SSLMigrationTest, TestFieldCacheSort, TestRetrieveFieldsOptimizer, TestBM25SimilarityFactory, TestJettySolrRunner, TestUseDocValuesAsStored, DistributedQueryComponentOptimizationTest, AddReplicaTest, TestXmlQParser, SynonymTokenizerTest, TestDistributedGrouping, TestHashPartitioner, TestRealTimeGet, TestCoreContainer, TestGroupingSearch, PeerSyncTest, TestSort, BasicFunctionalityTest, DistributedQueryElevationComponentTest, CoreAdminHandlerTest, TestCSVLoader, NoCacheHeaderTest, SolrCoreCheckLockOnStartupTest, QueryEqualityTest, TestRemoteStreaming, TestSolrDeletionPolicy1, SolrInfoBeanTest, IndexBasedSpellCheckerTest, DisMaxRequestHandlerTest, DocumentAnalysisRequestHandlerTest, RequestHandlersTest, PathHierarchyTokenizerFactoryTest, FieldAnalysisRequestHandlerTest, LoggingHandlerTest, IndexSchemaTest, QueryParsingTest, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, AlternateDirectoryTest, TestSolrIndexConfig, CopyFieldTest, TestSolrDeletionPolicy2, NumericFieldsTest, OutputWriterTest, DirectSolrConnectionTest, TestPhraseSuggestions, TestPerFieldSimilarity, PrimUtilsTest, SpellingQueryConverterTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestUtils, SystemInfoHandlerTest, DistributedIntervalFacetingTest, TestSimpleTrackingShardHandler, BaseCdcrDistributedZkTest, CdcrReplicationDistributedZkTest, CdcrReplicationHandlerTest, ChaosMonkeySafeLeaderWithPullReplicasTest, DeleteInactiveReplicaTest, DistribJoinFromCollectionTest, ForceLeaderTest]
[junit4] Completed [531/745 (1!)] on J1 in 156.01s, 3 tests, 1 error, 1 skipped <<< FAILURES!
[...truncated 47937 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 195 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/195/
4 tests failed.
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
expected:<0> but was:<1>
Stack Trace:
java.lang.AssertionError: expected:<0> but was:<1>
at __randomizedtesting.SeedInfo.seed([F937FCE697463010:7163C33C39BA5DE8]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:202)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.schema.TestCloudSchemaless.test
Error Message:
Stack Trace:
java.lang.NullPointerException
at __randomizedtesting.SeedInfo.seed([F937FCE697463010:7163C33C39BA5DE8]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:925)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:106)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:71)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:85)
at org.apache.solr.schema.TestCloudSchemaless.test(TestCloudSchemaless.java:171)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.update.TestInPlaceUpdatesDistrib.test
Error Message:
inplace_updatable_float didn't match for replica at client: https://127.0.0.1:36896/_/collection1 expected:<102.0> but was:<101.0>
Stack Trace:
java.lang.AssertionError: inplace_updatable_float didn't match for replica at client: https://127.0.0.1:36896/_/collection1 expected:<102.0> but was:<101.0>
at __randomizedtesting.SeedInfo.seed([F937FCE697463010:7163C33C39BA5DE8]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.apache.solr.update.TestInPlaceUpdatesDistrib.assertReplicaValue(TestInPlaceUpdatesDistrib.java:980)
at org.apache.solr.update.TestInPlaceUpdatesDistrib.delayedReorderingFetchesMissingUpdateFromLeaderTest(TestInPlaceUpdatesDistrib.java:875)
at org.apache.solr.update.TestInPlaceUpdatesDistrib.test(TestInPlaceUpdatesDistrib.java:145)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory
Error Message:
expected:<8> but was:<10>
Stack Trace:
java.lang.AssertionError: expected:<8> but was:<10>
at __randomizedtesting.SeedInfo.seed([F937FCE697463010:94CB581B2D0ECF17]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:262)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12243 lines...]
[junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/init-core-data-001
[junit4] 2> 1984751 INFO (SUITE-PeerSyncReplicationTest-seed#[F937FCE697463010]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1984752 INFO (SUITE-PeerSyncReplicationTest-seed#[F937FCE697463010]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 1984752 INFO (SUITE-PeerSyncReplicationTest-seed#[F937FCE697463010]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 1984752 INFO (SUITE-PeerSyncReplicationTest-seed#[F937FCE697463010]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 1984771 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1984778 INFO (Thread-2895) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1984778 INFO (Thread-2895) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1984778 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.ZkTestServer start zk server on port:33010
[junit4] 2> 1984884 ERROR (Thread-2895) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1985226 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1985227 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 1985228 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1985229 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1985242 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1985242 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1985243 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1985243 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1985244 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1985244 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1985245 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1985279 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f5d0a2bca0001, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1985444 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 1986250 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1986341 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@191ee19f{/,null,AVAILABLE}
[junit4] 2> 1986342 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@312b369f{SSL,[ssl, http/1.1]}{127.0.0.1:33900}
[junit4] 2> 1986342 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server Started @2002731ms
[junit4] 2> 1986342 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/tempDir-001/control/data, hostContext=/, hostPort=33900, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/control-001/cores}
[junit4] 2> 1986342 ERROR (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1986342 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 1986342 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1986342 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1986342 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-27T08:54:30.065Z
[junit4] 2> 1986446 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1986446 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/control-001/solr.xml
[junit4] 2> 1986496 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1986496 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1986497 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1986568 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33010/solr
[junit4] 2> 1986644 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f5d0a2bca0004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1988484 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1988485 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33900_
[junit4] 2> 1988485 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.c.Overseer Overseer (id=98900015091154949-127.0.0.1:33900_-n_0000000000) starting
[junit4] 2> 1988537 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33900_
[junit4] 2> 1988598 INFO (OverseerStateUpdate-98900015091154949-127.0.0.1:33900_-n_0000000000) [n:127.0.0.1:33900_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1989050 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1989103 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1989103 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1989104 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33900_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/control-001/cores
[junit4] 2> 1989436 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1989436 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33010/solr ready
[junit4] 2> 1989730 INFO (qtp882637231-6693) [n:127.0.0.1:33900_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:33900_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1989781 INFO (OverseerThreadFactory-2236-thread-1-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 1989981 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1989982 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1990091 INFO (zkCallback-784-thread-1-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1991287 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 1991413 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 1992262 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1992351 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 1992351 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1992351 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1992351 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 1992386 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=32.7412109375, floorSegmentMB=0.9931640625, forceMergeDeletesPctAllowed=4.645882138600275, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4070763039360047
[junit4] 2> 1992462 WARN (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1992736 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1992736 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1992737 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1992737 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1992739 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=16.9990234375, floorSegmentMB=1.7177734375, forceMergeDeletesPctAllowed=29.80170751436371, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.25869791656211993
[junit4] 2> 1992739 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@63671baf[control_collection_shard1_replica_n1] main]
[junit4] 2> 1992740 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1992753 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1992753 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1992753 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582400249765298176
[junit4] 2> 1992774 INFO (searcherExecutor-2239-thread-1-processing-n:127.0.0.1:33900_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@63671baf[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1992775 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1992775 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1992775 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:33900/control_collection_shard1_replica_n1/
[junit4] 2> 1992775 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1992775 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:33900/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1992775 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1992789 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:33900/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 1992790 INFO (zkCallback-784-thread-2-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1992846 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1992847 INFO (qtp882637231-6690) [n:127.0.0.1:33900_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2865
[junit4] 2> 1992849 INFO (qtp882637231-6693) [n:127.0.0.1:33900_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1992956 INFO (zkCallback-784-thread-2-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1993799 INFO (OverseerCollectionConfigSetProcessor-98900015091154949-127.0.0.1:33900_-n_0000000000) [n:127.0.0.1:33900_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1993856 INFO (qtp882637231-6693) [n:127.0.0.1:33900_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:33900_&wt=javabin&version=2} status=0 QTime=4126
[junit4] 2> 1993974 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1994003 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33010/solr ready
[junit4] 2> 1994003 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1994018 INFO (qtp882637231-6694) [n:127.0.0.1:33900_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1994092 INFO (OverseerThreadFactory-2236-thread-2-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1994093 WARN (OverseerThreadFactory-2236-thread-2-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 1994397 INFO (qtp882637231-6694) [n:127.0.0.1:33900_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1994397 INFO (qtp882637231-6694) [n:127.0.0.1:33900_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=378
[junit4] 2> 1995416 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-1-001 of type NRT
[junit4] 2> 1995430 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1995490 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@43f243f0{/,null,AVAILABLE}
[junit4] 2> 1995491 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3f094b93{SSL,[ssl, http/1.1]}{127.0.0.1:33495}
[junit4] 2> 1995491 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server Started @2011880ms
[junit4] 2> 1995491 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=33495, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-1-001/cores}
[junit4] 2> 1995491 ERROR (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1995491 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 1995491 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1995491 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 1995491 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-27T08:54:39.214Z
[junit4] 2> 1995566 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1995566 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-1-001/solr.xml
[junit4] 2> 1995602 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1995602 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1995603 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1995637 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33010/solr
[junit4] 2> 1995745 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33495_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1995746 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33495_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1995759 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33495_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33495_
[junit4] 2> 1995836 INFO (zkCallback-792-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1995963 INFO (zkCallback-797-thread-1-processing-n:127.0.0.1:33495_) [n:127.0.0.1:33495_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1995963 INFO (zkCallback-784-thread-1-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1996129 INFO (OverseerCollectionConfigSetProcessor-98900015091154949-127.0.0.1:33900_-n_0000000000) [n:127.0.0.1:33900_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1996276 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33495_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1996346 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33495_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1996346 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33495_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1996348 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:33495_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-1-001/cores
[junit4] 2> 1996622 INFO (qtp1900424130-6741) [n:127.0.0.1:33495_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:33495_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1996624 INFO (OverseerThreadFactory-2236-thread-3-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:33495_ for creating new replica
[junit4] 2> 1996734 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1996734 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1997868 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 1997994 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema [collection1_shard1_replica_n21] Schema name=test
[junit4] 2> 1999145 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1999275 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n21' using configuration from collection collection1, trusted=true
[junit4] 2> 1999275 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n21' (registry 'solr.core.collection1.shard1.replica_n21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 1999275 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1999275 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [[collection1_shard1_replica_n21] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-1-001/cores/collection1_shard1_replica_n21], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-1-001/cores/collection1_shard1_replica_n21/data/]
[junit4] 2> 1999347 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=32.7412109375, floorSegmentMB=0.9931640625, forceMergeDeletesPctAllowed=4.645882138600275, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4070763039360047
[junit4] 2> 1999386 WARN (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1999844 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1999844 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1999845 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1999845 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1999846 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=16.9990234375, floorSegmentMB=1.7177734375, forceMergeDeletesPctAllowed=29.80170751436371, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.25869791656211993
[junit4] 2> 1999847 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.SolrIndexSearcher Opening [Searcher@52595c65[collection1_shard1_replica_n21] main]
[junit4] 2> 1999848 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1999862 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1999863 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1999897 INFO (searcherExecutor-2250-thread-1-processing-n:127.0.0.1:33495_ x:collection1_shard1_replica_n21 s:shard1 c:collection1) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [collection1_shard1_replica_n21] Registered new searcher Searcher@52595c65[collection1_shard1_replica_n21] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1999897 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582400257256325120
[junit4] 2> 1999900 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1999900 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1999900 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:33495/collection1_shard1_replica_n21/
[junit4] 2> 1999900 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1999900 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy https://127.0.0.1:33495/collection1_shard1_replica_n21/ has no replicas
[junit4] 2> 1999900 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1999918 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:33495/collection1_shard1_replica_n21/ shard1
[junit4] 2> 1999970 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1999971 INFO (qtp1900424130-6735) [n:127.0.0.1:33495_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3237
[junit4] 2> 1999973 INFO (qtp1900424130-6741) [n:127.0.0.1:33495_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:33495_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=3350
[junit4] 2> 2000646 INFO (OverseerCollectionConfigSetProcessor-98900015091154949-127.0.0.1:33900_-n_0000000000) [n:127.0.0.1:33900_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2000724 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-2-001 of type NRT
[junit4] 2> 2000725 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2000886 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@14b74cae{/,null,AVAILABLE}
[junit4] 2> 2000886 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@504f126f{SSL,[ssl, http/1.1]}{127.0.0.1:35892}
[junit4] 2> 2000886 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server Started @2017275ms
[junit4] 2> 2000886 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=35892, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-2-001/cores}
[junit4] 2> 2000887 ERROR (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2000887 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 2000887 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2000887 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2000887 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-27T08:54:44.610Z
[junit4] 2> 2000975 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2000975 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-2-001/solr.xml
[junit4] 2> 2000979 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2000979 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2001012 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2001049 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33010/solr
[junit4] 2> 2001108 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:35892_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2001143 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:35892_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2001145 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:35892_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35892_
[junit4] 2> 2001145 INFO (zkCallback-792-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2001145 INFO (zkCallback-797-thread-1-processing-n:127.0.0.1:33495_) [n:127.0.0.1:33495_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2001158 INFO (zkCallback-784-thread-1-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2001216 INFO (zkCallback-803-thread-1-processing-n:127.0.0.1:35892_) [n:127.0.0.1:35892_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2002031 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:35892_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 2002127 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:35892_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 2002127 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:35892_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 2002129 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:35892_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-2-001/cores
[junit4] 2> 2002733 INFO (qtp1900424130-6734) [n:127.0.0.1:33495_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:35892_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2002735 INFO (OverseerThreadFactory-2236-thread-4-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:35892_ for creating new replica
[junit4] 2> 2003168 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2003168 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2004304 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2004466 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.IndexSchema [collection1_shard1_replica_n23] Schema name=test
[junit4] 2> 2005369 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2005408 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n23' using configuration from collection collection1, trusted=true
[junit4] 2> 2005408 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n23' (registry 'solr.core.collection1.shard1.replica_n23') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 2005408 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2005408 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrCore [[collection1_shard1_replica_n23] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-2-001/cores/collection1_shard1_replica_n23], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-2-001/cores/collection1_shard1_replica_n23/data/]
[junit4] 2> 2005444 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=32.7412109375, floorSegmentMB=0.9931640625, forceMergeDeletesPctAllowed=4.645882138600275, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4070763039360047
[junit4] 2> 2005508 WARN (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2005984 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2005984 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2005985 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2005985 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2005986 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=16.9990234375, floorSegmentMB=1.7177734375, forceMergeDeletesPctAllowed=29.80170751436371, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.25869791656211993
[junit4] 2> 2006021 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.SolrIndexSearcher Opening [Searcher@cd9f940[collection1_shard1_replica_n23] main]
[junit4] 2> 2006022 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2006023 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2006057 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2006059 INFO (searcherExecutor-2261-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrCore [collection1_shard1_replica_n23] Registered new searcher Searcher@cd9f940[collection1_shard1_replica_n23] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2006059 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582400263717650432
[junit4] 2> 2006116 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.ZkController Core needs to recover:collection1_shard1_replica_n23
[junit4] 2> 2006116 INFO (qtp301720388-6770) [n:127.0.0.1:35892_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2997
[junit4] 2> 2006131 INFO (qtp1900424130-6734) [n:127.0.0.1:33495_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:35892_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=3398
[junit4] 2> 2006133 INFO (updateExecutor-800-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2006133 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2006194 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2006195 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard1_replica_n23]
[junit4] 2> 2006195 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2006195 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard1_replica_n23] as recovering, leader is [https://127.0.0.1:33495/collection1_shard1_replica_n21/] and I am [https://127.0.0.1:35892/collection1_shard1_replica_n23/]
[junit4] 2> 2006236 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Sending prep recovery command to [https://127.0.0.1:33495]; [WaitForState: action=PREPRECOVERY&core=collection1_shard1_replica_n21&nodeName=127.0.0.1:35892_&coreNodeName=core_node24&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2006272 INFO (qtp1900424130-6738) [n:127.0.0.1:33495_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node24, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
[junit4] 2> 2006272 INFO (qtp1900424130-6738) [n:127.0.0.1:33495_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_n21, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:35892_, coreNodeName=core_node24, onlyIfActiveCheckResult=false, nodeProps: core_node24:{"core":"collection1_shard1_replica_n23","base_url":"https://127.0.0.1:35892","node_name":"127.0.0.1:35892_","state":"recovering","type":"NRT"}
[junit4] 2> 2006272 INFO (qtp1900424130-6738) [n:127.0.0.1:33495_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node24, state: recovering, checkLive: true, onlyIfLeader: true for: 0 seconds.
[junit4] 2> 2006272 INFO (qtp1900424130-6738) [n:127.0.0.1:33495_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:35892_&onlyIfLeaderActive=true&core=collection1_shard1_replica_n21&coreNodeName=core_node24&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2006777 INFO (OverseerCollectionConfigSetProcessor-98900015091154949-127.0.0.1:33900_-n_0000000000) [n:127.0.0.1:33900_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2006971 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [https://127.0.0.1:33495/collection1_shard1_replica_n21/] - recoveringAfterStartup=[true]
[junit4] 2> 2006972 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_n23 url=https://127.0.0.1:35892 START replicas=[https://127.0.0.1:33495/collection1_shard1_replica_n21/] nUpdates=1000
[junit4] 2> 2007102 INFO (qtp1900424130-6739) [n:127.0.0.1:33495_ c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_n21] o.a.s.u.IndexFingerprint IndexFingerprint millis:56.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2007102 INFO (qtp1900424130-6739) [n:127.0.0.1:33495_ c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_n21] o.a.s.c.S.Request [collection1_shard1_replica_n21] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=86
[junit4] 2> 2007119 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2007119 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2007119 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2007119 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2007191 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2007191 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2007191 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2007191 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2007192 INFO (recoveryExecutor-801-thread-1-processing-n:127.0.0.1:35892_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:35892_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2007268 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-3-001 of type NRT
[junit4] 2> 2007317 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2007427 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@20a6a8d0{/,null,AVAILABLE}
[junit4] 2> 2007430 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@76aeb631{SSL,[ssl, http/1.1]}{127.0.0.1:41808}
[junit4] 2> 2007430 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.e.j.s.Server Started @2023819ms
[junit4] 2> 2007430 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=41808, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-3-001/cores}
[junit4] 2> 2007430 ERROR (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2007447 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 2007448 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2007448 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2007448 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-27T08:54:51.171Z
[junit4] 2> 2007533 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2007533 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-3-001/solr.xml
[junit4] 2> 2007536 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2007536 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2007570 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2007628 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33010/solr
[junit4] 2> 2007685 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:41808_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2007686 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:41808_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2007687 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:41808_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41808_
[junit4] 2> 2007720 INFO (zkCallback-784-thread-1-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2007793 INFO (zkCallback-810-thread-1-processing-n:127.0.0.1:41808_) [n:127.0.0.1:41808_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2007793 INFO (zkCallback-803-thread-1-processing-n:127.0.0.1:35892_) [n:127.0.0.1:35892_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2007720 INFO (zkCallback-792-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2007821 INFO (zkCallback-797-thread-1-processing-n:127.0.0.1:33495_) [n:127.0.0.1:33495_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2008401 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:41808_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 2008563 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:41808_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 2008563 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:41808_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@245c85f6
[junit4] 2> 2008564 INFO (TEST-PeerSyncReplicationTest.test-seed#[F937FCE697463010]) [n:127.0.0.1:41808_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_F937FCE697463010-001/shard-3-001/cores
[junit4] 2> 2008856 INFO (qtp1900424130-6740) [n:127.0.0.1:33495_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41808_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2008892 INFO (OverseerThreadFactory-2236-thread-5-processing-n:127.0.0.1:33900_) [n:127.0.0.1:33900_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:41808_ for creating new replica
[junit4] 2> 2009207 INFO (qtp950198755-6802) [n:127.0.0.1:41808_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n25&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2009207 INFO (qtp950198755-6802) [n:127.0.0.1:41808_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2010447 INFO (qtp950198755-6802) [n:127.0.0.1:41808_ c:collection1 s:shard1 x:collection1_shard1_replica_n25] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2010722 INFO (qtp950198755-6802) [n:127.0.0.1:41808_ c:collection1 s:shard1 x:collection1_shard1_rep
[...truncated too long message...]
ng() + ": " + e.getMessage(), e);
[ecj-lint] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: 'client' is not closed at this location
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 11. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java (at line 204)
[ecj-lint] Analyzer a1 = new WhitespaceAnalyzer();
[ecj-lint] ^^
[ecj-lint] Resource leak: 'a1' is never closed
[ecj-lint] ----------
[ecj-lint] 12. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java (at line 207)
[ecj-lint] OffsetWindowTokenFilter tots = new OffsetWindowTokenFilter(tokenStream);
[ecj-lint] ^^^^
[ecj-lint] Resource leak: 'tots' is never closed
[ecj-lint] ----------
[ecj-lint] 13. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java (at line 211)
[ecj-lint] Analyzer a2 = new WhitespaceAnalyzer();
[ecj-lint] ^^
[ecj-lint] Resource leak: 'a2' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 14. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/response/TestJavabinTupleStreamParser.java (at line 72)
[ecj-lint] JavabinTupleStreamParser parser = new JavabinTupleStreamParser(new ByteArrayInputStream(bytes), true);
[ecj-lint] ^^^^^^
[ecj-lint] Resource leak: 'parser' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 15. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/search/TestDocSet.java (at line 243)
[ecj-lint] return loadfactor!=0 ? new HashDocSet(a,0,n,1/loadfactor) : new HashDocSet(a,0,n);
[ecj-lint] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
[ecj-lint] ----------
[ecj-lint] 16. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/search/TestDocSet.java (at line 528)
[ecj-lint] DocSet a = new BitDocSet(bs);
[ecj-lint] ^
[ecj-lint] Resource leak: 'a' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 17. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/security/TestPKIAuthenticationPlugin.java (at line 76)
[ecj-lint] final MockPKIAuthenticationPlugin mock = new MockPKIAuthenticationPlugin(null, nodeName);
[ecj-lint] ^^^^
[ecj-lint] Resource leak: 'mock' is never closed
[ecj-lint] ----------
[ecj-lint] 18. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/security/TestPKIAuthenticationPlugin.java (at line 131)
[ecj-lint] MockPKIAuthenticationPlugin mock1 = new MockPKIAuthenticationPlugin(null, nodeName) {
[ecj-lint] ^^^^^
[ecj-lint] Resource leak: 'mock1' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 19. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/security/TestRuleBasedAuthorizationPlugin.java (at line 380)
[ecj-lint] RuleBasedAuthorizationPlugin plugin = new RuleBasedAuthorizationPlugin();
[ecj-lint] ^^^^^^
[ecj-lint] Resource leak: 'plugin' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 20. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/security/TestSha256AuthenticationProvider.java (at line 49)
[ecj-lint] BasicAuthPlugin basicAuthPlugin = new BasicAuthPlugin();
[ecj-lint] ^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: 'basicAuthPlugin' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 21. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java (at line 42)
[ecj-lint] WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer();
[ecj-lint] ^^^^^^^^
[ecj-lint] Resource leak: 'analyzer' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 22. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java (at line 139)
[ecj-lint] IndexWriter w = new IndexWriter(d, newIndexWriterConfig(analyzer));
[ecj-lint] ^
[ecj-lint] Resource leak: 'w' is never closed
[ecj-lint] ----------
[ecj-lint] 23. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java (at line 172)
[ecj-lint] throw iae;
[ecj-lint] ^^^^^^^^^^
[ecj-lint] Resource leak: 'w' is not closed at this location
[ecj-lint] ----------
[ecj-lint] 24. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java (at line 178)
[ecj-lint] return;
[ecj-lint] ^^^^^^^
[ecj-lint] Resource leak: 'w' is not closed at this location
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 25. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java (at line 134)
[ecj-lint] SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
[ecj-lint] ^^^^^^^^^^
[ecj-lint] Resource leak: 'cmdDistrib' is never closed
[ecj-lint] ----------
[ecj-lint] 26. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java (at line 333)
[ecj-lint] SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 5, 0);
[ecj-lint] ^^^^^^^^^^
[ecj-lint] Resource leak: 'cmdDistrib' is never closed
[ecj-lint] ----------
[ecj-lint] 27. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java (at line 367)
[ecj-lint] SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 5, 0);
[ecj-lint] ^^^^^^^^^^
[ecj-lint] Resource leak: 'cmdDistrib' is never closed
[ecj-lint] ----------
[ecj-lint] 28. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java (at line 413)
[ecj-lint] SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 5, 0);
[ecj-lint] ^^^^^^^^^^
[ecj-lint] Resource leak: 'cmdDistrib' is never closed
[ecj-lint] ----------
[ecj-lint] 29. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java (at line 458)
[ecj-lint] SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
[ecj-lint] ^^^^^^^^^^
[ecj-lint] Resource leak: 'cmdDistrib' is never closed
[ecj-lint] ----------
[ecj-lint] 30. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java (at line 516)
[ecj-lint] SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
[ecj-lint] ^^^^^^^^^^
[ecj-lint] Resource leak: 'cmdDistrib' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 31. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java (at line 181)
[ecj-lint] EmbeddedSolrServer server1 = new EmbeddedSolrServer(h.getCoreContainer(), "split1");
[ecj-lint] ^^^^^^^
[ecj-lint] Resource leak: 'server1' is never closed
[ecj-lint] ----------
[ecj-lint] 32. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java (at line 182)
[ecj-lint] EmbeddedSolrServer server2 = new EmbeddedSolrServer(h.getCoreContainer(), "split2");
[ecj-lint] ^^^^^^^
[ecj-lint] Resource leak: 'server2' is never closed
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 33. WARNING in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test/org/apache/solr/update/processor/RecordingUpdateProcessorFactory.java (at line 67)
[ecj-lint] return recording ? new RecordingUpdateRequestProcessor(commandQueue, next) : next;
[ecj-lint] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
[ecj-lint] ----------
[ecj-lint] 33 problems (1 error, 32 warnings)
BUILD FAILED
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:826: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:101: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build.xml:684: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:2093: The following error occurred while executing this line:
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:2126: Compile failed; see the compiler error output for details.
Total time: 344 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 194 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/194/
6 tests failed.
FAILED: org.apache.solr.cloud.HttpPartitionTest.test
Error Message:
The partitioned replica did not get marked down expected:<[down]> but was:<[active]>
Stack Trace:
org.junit.ComparisonFailure: The partitioned replica did not get marked down expected:<[down]> but was:<[active]>
at __randomizedtesting.SeedInfo.seed([4D0F748AACCA44B0:C55B4B5002362948]:0)
at org.junit.Assert.assertEquals(Assert.java:125)
at org.apache.solr.cloud.HttpPartitionTest.testMinRf(HttpPartitionTest.java:241)
at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:127)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestCloudRecovery.leaderRecoverFromLogOnStartupTest
Error Message:
Stack Trace:
java.lang.NullPointerException
at __randomizedtesting.SeedInfo.seed([4D0F748AACCA44B0:39FF95CFE8EE033F]:0)
at org.apache.solr.cloud.TestCloudRecovery.leaderRecoverFromLogOnStartupTest(TestCloudRecovery.java:124)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration
Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:50483_solr wasn't created
Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:50483_solr wasn't created
at __randomizedtesting.SeedInfo.seed([4D0F748AACCA44B0:55B5FC86A2FF895F]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration(TriggerIntegrationTest.java:909)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.handler.TestSQLHandler.doTest
Error Message:
Error from server at http://127.0.0.1:43468: KeeperErrorCode = Session expired for /overseer/collection-queue-work/qnr-
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:43468: KeeperErrorCode = Session expired for /overseer/collection-queue-work/qnr-
at __randomizedtesting.SeedInfo.seed([4D0F748AACCA44B0:EA4BCC2EC1715709]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.reloadCollection(AbstractFullDistribZkTestBase.java:2045)
at org.apache.solr.handler.TestSQLHandler.testBasicSelect(TestSQLHandler.java:306)
at org.apache.solr.handler.TestSQLHandler.doTest(TestSQLHandler.java:82)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory
Error Message:
replica never fully recovered
Stack Trace:
java.lang.AssertionError: replica never fully recovered
at __randomizedtesting.SeedInfo.seed([4D0F748AACCA44B0:20F3D0771682BBB7]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.waitForRecovery(AutoscalingHistoryHandlerTest.java:303)
at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:255)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.search.join.BlockJoinFacetDistribTest.testBJQFacetComponent
Error Message:
Could not load collection from ZK: facetcollection
Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: facetcollection
at __randomizedtesting.SeedInfo.seed([4D0F748AACCA44B0:1DF2C4F727369138]:0)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1188)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:832)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
at org.apache.solr.search.join.BlockJoinFacetDistribTest.testBJQFacetComponent(BlockJoinFacetDistribTest.java:91)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/facetcollection/state.json
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
... 47 more
Build Log:
[...truncated 12564 lines...]
[junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/init-core-data-001
[junit4] 2> 3334387 WARN (SUITE-HttpPartitionTest-seed#[4D0F748AACCA44B0]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=24 numCloses=24
[junit4] 2> 3334387 INFO (SUITE-HttpPartitionTest-seed#[4D0F748AACCA44B0]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 3334524 INFO (SUITE-HttpPartitionTest-seed#[4D0F748AACCA44B0]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 3334524 INFO (SUITE-HttpPartitionTest-seed#[4D0F748AACCA44B0]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 3334524 INFO (SUITE-HttpPartitionTest-seed#[4D0F748AACCA44B0]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 3334532 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3334576 INFO (Thread-1783) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3334576 INFO (Thread-1783) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3334632 ERROR (Thread-1783) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3334708 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.ZkTestServer start zk server on port:43744
[junit4] 2> 3334762 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f5a42748a0000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3335269 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 3335510 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 3335571 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 3335588 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 3335592 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 3335596 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 3335600 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 3335608 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 3335609 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 3335609 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 3335610 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 3335620 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 3336750 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3336753 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@23c008fd{/,null,AVAILABLE}
[junit4] 2> 3336753 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@27375f42{HTTP/1.1,[http/1.1]}{127.0.0.1:52638}
[junit4] 2> 3336753 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server Started @3358194ms
[junit4] 2> 3336753 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/tempDir-001/control/data, hostContext=/, hostPort=59675, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/control-001/cores}
[junit4] 2> 3336754 ERROR (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3336754 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 3336754 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3336754 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 3336754 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-26T19:57:07.712Z
[junit4] 2> 3336970 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3336971 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/control-001/solr.xml
[junit4] 2> 3336982 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3336982 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3336983 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3337033 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43744/solr
[junit4] 2> 3339264 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3339268 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59675_
[junit4] 2> 3339276 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.c.Overseer Overseer (id=98896958294982660-127.0.0.1:59675_-n_0000000000) starting
[junit4] 2> 3339620 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59675_
[junit4] 2> 3339624 INFO (zkCallback-1153-thread-1-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3340493 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3340595 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3340596 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3340675 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:59675_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/control-001/cores
[junit4] 2> 3342438 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3342438 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43744/solr ready
[junit4] 2> 3342457 INFO (SocketProxy-Acceptor-59675) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54939,localport=59675], receiveBufferSize:531000
[junit4] 2> 3342552 INFO (SocketProxy-Acceptor-59675) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=52638,localport=48610], receiveBufferSize=530904
[junit4] 2> 3342700 INFO (qtp1321684759-7417) [n:127.0.0.1:59675_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:59675_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3342835 INFO (OverseerThreadFactory-3201-thread-1-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 3342985 INFO (SocketProxy-Acceptor-59675) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54941,localport=59675], receiveBufferSize:531000
[junit4] 2> 3343000 INFO (SocketProxy-Acceptor-59675) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=52638,localport=48612], receiveBufferSize=530904
[junit4] 2> 3343073 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3343073 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3343372 INFO (zkCallback-1153-thread-1-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3344374 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 3344719 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 3345835 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3346017 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 3346017 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3346081 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3346081 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 3346264 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=30.1689453125, floorSegmentMB=1.142578125, forceMergeDeletesPctAllowed=24.11397041105384, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24045656329513784
[junit4] 2> 3346270 WARN (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3348508 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3348508 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3348517 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3348517 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3348611 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=96.0751953125, floorSegmentMB=0.677734375, forceMergeDeletesPctAllowed=27.557608195764637, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3967483425731648
[junit4] 2> 3348611 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@65ddea9a[control_collection_shard1_replica_n1] main]
[junit4] 2> 3348612 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3348803 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3348804 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3348807 INFO (searcherExecutor-3204-thread-1-processing-n:127.0.0.1:59675_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@65ddea9a[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3348807 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582351347629424640
[junit4] 2> 3348820 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3348820 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3348820 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:59675/control_collection_shard1_replica_n1/
[junit4] 2> 3348820 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3348820 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:59675/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 3348820 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3348824 INFO (zkCallback-1153-thread-2-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3348824 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:59675/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 3348825 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3348826 INFO (qtp1321684759-7415) [n:127.0.0.1:59675_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=5752
[junit4] 2> 3348830 INFO (qtp1321684759-7417) [n:127.0.0.1:59675_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3348954 INFO (zkCallback-1153-thread-2-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3349240 INFO (OverseerCollectionConfigSetProcessor-98896958294982660-127.0.0.1:59675_-n_0000000000) [n:127.0.0.1:59675_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3349838 INFO (qtp1321684759-7417) [n:127.0.0.1:59675_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:59675_&wt=javabin&version=2} status=0 QTime=7137
[junit4] 2> 3352201 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3352202 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43744/solr ready
[junit4] 2> 3352202 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 3352202 INFO (SocketProxy-Acceptor-59675) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=54947,localport=59675], receiveBufferSize:531000
[junit4] 2> 3352282 INFO (SocketProxy-Acceptor-59675) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=52638,localport=48618], receiveBufferSize=530904
[junit4] 2> 3352322 INFO (qtp1321684759-7418) [n:127.0.0.1:59675_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3352449 INFO (OverseerThreadFactory-3201-thread-2-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 3352450 WARN (OverseerThreadFactory-3201-thread-2-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 3352886 INFO (qtp1321684759-7418) [n:127.0.0.1:59675_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3352886 INFO (qtp1321684759-7418) [n:127.0.0.1:59675_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=563
[junit4] 2> 3353607 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-1-001 of type TLOG
[junit4] 2> 3353608 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3354396 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1fb6a4d7{/,null,AVAILABLE}
[junit4] 2> 3354396 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3583ebd{HTTP/1.1,[http/1.1]}{127.0.0.1:39669}
[junit4] 2> 3354396 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server Started @3375836ms
[junit4] 2> 3354396 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=56283, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-1-001/cores}
[junit4] 2> 3354397 ERROR (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3354397 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 3354397 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3354397 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 3354397 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-26T19:57:25.355Z
[junit4] 2> 3354451 INFO (OverseerCollectionConfigSetProcessor-98896958294982660-127.0.0.1:59675_-n_0000000000) [n:127.0.0.1:59675_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3357976 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3357976 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-1-001/solr.xml
[junit4] 2> 3358013 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3358013 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3358015 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3358136 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43744/solr
[junit4] 2> 3359143 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:56283_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3359144 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:56283_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3359146 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:56283_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56283_
[junit4] 2> 3359374 INFO (zkCallback-1166-thread-2-processing-n:127.0.0.1:56283_) [n:127.0.0.1:56283_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3359620 INFO (zkCallback-1153-thread-2-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3359745 INFO (zkCallback-1161-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3359963 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:56283_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3360111 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:56283_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3360111 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:56283_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3360279 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:56283_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-1-001/cores
[junit4] 2> 3360528 INFO (SocketProxy-Acceptor-56283) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=39961,localport=56283], receiveBufferSize:531000
[junit4] 2> 3360608 INFO (SocketProxy-Acceptor-56283) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=39669,localport=42583], receiveBufferSize=530904
[junit4] 2> 3360718 INFO (qtp1771810288-7471) [n:127.0.0.1:56283_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:56283_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3360745 INFO (OverseerThreadFactory-3201-thread-3-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:56283_ for creating new replica
[junit4] 2> 3360748 INFO (SocketProxy-Acceptor-56283) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=39963,localport=56283], receiveBufferSize:531000
[junit4] 2> 3360749 INFO (SocketProxy-Acceptor-56283) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=39669,localport=42585], receiveBufferSize=530904
[junit4] 2> 3360793 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 3360793 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3360939 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 3361071 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema [collection1_shard2_replica_t41] Schema name=test
[junit4] 2> 3361962 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3361973 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t41' using configuration from collection collection1, trusted=true
[junit4] 2> 3361974 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t41' (registry 'solr.core.collection1.shard2.replica_t41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3361974 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3361974 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [[collection1_shard2_replica_t41] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-1-001/cores/collection1_shard2_replica_t41], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-1-001/cores/collection1_shard2_replica_t41/data/]
[junit4] 2> 3361976 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=30.1689453125, floorSegmentMB=1.142578125, forceMergeDeletesPctAllowed=24.11397041105384, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24045656329513784
[junit4] 2> 3361978 WARN (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3362010 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3362010 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3362011 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3362011 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3362013 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=96.0751953125, floorSegmentMB=0.677734375, forceMergeDeletesPctAllowed=27.557608195764637, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3967483425731648
[junit4] 2> 3362013 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.SolrIndexSearcher Opening [Searcher@53efa664[collection1_shard2_replica_t41] main]
[junit4] 2> 3362014 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3362014 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3362014 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3362015 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582351361479016448
[junit4] 2> 3362018 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3362018 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3362018 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56283/collection1_shard2_replica_t41/
[junit4] 2> 3362018 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3362018 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy http://127.0.0.1:56283/collection1_shard2_replica_t41/ has no replicas
[junit4] 2> 3362018 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3362019 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController collection1_shard2_replica_t41 stopping background replication from leader
[junit4] 2> 3362021 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56283/collection1_shard2_replica_t41/ shard2
[junit4] 2> 3362022 INFO (searcherExecutor-3215-thread-1-processing-n:127.0.0.1:56283_ x:collection1_shard2_replica_t41 s:shard2 c:collection1) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [collection1_shard2_replica_t41] Registered new searcher Searcher@53efa664[collection1_shard2_replica_t41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3362172 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3362173 INFO (qtp1771810288-7469) [n:127.0.0.1:56283_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1379
[junit4] 2> 3362175 INFO (qtp1771810288-7471) [n:127.0.0.1:56283_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:56283_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=1456
[junit4] 2> 3362748 INFO (OverseerCollectionConfigSetProcessor-98896958294982660-127.0.0.1:59675_-n_0000000000) [n:127.0.0.1:59675_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3363089 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-2-001 of type TLOG
[junit4] 2> 3363090 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3363416 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2042af88{/,null,AVAILABLE}
[junit4] 2> 3363417 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@34d81f48{HTTP/1.1,[http/1.1]}{127.0.0.1:41685}
[junit4] 2> 3363417 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server Started @3384857ms
[junit4] 2> 3363417 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=54481, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-2-001/cores}
[junit4] 2> 3363417 ERROR (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3363424 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 3363424 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3363424 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 3363424 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-26T19:57:34.382Z
[junit4] 2> 3363611 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3363611 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-2-001/solr.xml
[junit4] 2> 3363614 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3363615 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3363616 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3363622 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43744/solr
[junit4] 2> 3364972 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:54481_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3364974 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:54481_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3364989 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:54481_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54481_
[junit4] 2> 3365009 INFO (zkCallback-1153-thread-1-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3365009 INFO (zkCallback-1161-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3365009 INFO (zkCallback-1166-thread-2-processing-n:127.0.0.1:56283_) [n:127.0.0.1:56283_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3365226 INFO (zkCallback-1172-thread-1-processing-n:127.0.0.1:54481_) [n:127.0.0.1:54481_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3366416 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:54481_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3366673 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:54481_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3366673 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:54481_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3366674 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [n:127.0.0.1:54481_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-2-001/cores
[junit4] 2> 3367061 INFO (SocketProxy-Acceptor-54481) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=33414,localport=54481], receiveBufferSize:531000
[junit4] 2> 3367129 INFO (SocketProxy-Acceptor-54481) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=41685,localport=50630], receiveBufferSize=530904
[junit4] 2> 3367161 INFO (qtp239820775-7507) [n:127.0.0.1:54481_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:54481_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3367461 INFO (OverseerThreadFactory-3201-thread-4-processing-n:127.0.0.1:59675_) [n:127.0.0.1:59675_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:54481_ for creating new replica
[junit4] 2> 3367512 INFO (SocketProxy-Acceptor-54481) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=33416,localport=54481], receiveBufferSize:531000
[junit4] 2> 3367512 INFO (SocketProxy-Acceptor-54481) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=41685,localport=50632], receiveBufferSize=530904
[junit4] 2> 3367591 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 3367591 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3368781 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 3368932 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema [collection1_shard1_replica_t43] Schema name=test
[junit4] 2> 3369891 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3370028 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t43' using configuration from collection collection1, trusted=true
[junit4] 2> 3370029 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t43' (registry 'solr.core.collection1.shard1.replica_t43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@416508dd
[junit4] 2> 3370029 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3370029 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [[collection1_shard1_replica_t43] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-2-001/cores/collection1_shard1_replica_t43], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-2-001/cores/collection1_shard1_replica_t43/data/]
[junit4] 2> 3370076 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=30.1689453125, floorSegmentMB=1.142578125, forceMergeDeletesPctAllowed=24.11397041105384, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24045656329513784
[junit4] 2> 3370118 WARN (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3372388 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3372388 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3372453 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3372453 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3372480 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=96.0751953125, floorSegmentMB=0.677734375, forceMergeDeletesPctAllowed=27.557608195764637, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3967483425731648
[junit4] 2> 3372480 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.SolrIndexSearcher Opening [Searcher@29b39e83[collection1_shard1_replica_t43] main]
[junit4] 2> 3372481 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3372481 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3372482 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3372482 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582351372454461440
[junit4] 2> 3372493 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3372493 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3372493 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:54481/collection1_shard1_replica_t43/
[junit4] 2> 3372493 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3372493 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy http://127.0.0.1:54481/collection1_shard1_replica_t43/ has no replicas
[junit4] 2> 3372494 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3372504 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController collection1_shard1_replica_t43 stopping background replication from leader
[junit4] 2> 3372512 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:54481/collection1_shard1_replica_t43/ shard1
[junit4] 2> 3372512 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3372512 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController No registered searcher found for core: collection1_shard1_replica_t43, waiting until a searcher is registered before publishing as active
[junit4] 2> 3372528 INFO (searcherExecutor-3226-thread-1-processing-n:127.0.0.1:54481_ x:collection1_shard1_replica_t43 s:shard1 c:collection1) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [collection1_shard1_replica_t43] Registered new searcher Searcher@29b39e83[collection1_shard1_replica_t43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3372529 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController Found a registered searcher: Searcher@29b39e83[collection1_shard1_replica_t43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}, took: 16.0 ms for core: collection1_shard1_replica_t43, id: org.apache.solr.core.SolrCore@453bc184
[junit4] 2> 3372588 INFO (qtp239820775-7505) [n:127.0.0.1:54481_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=4997
[junit4] 2> 3372590 INFO (qtp239820775-7507) [n:127.0.0.1:54481_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:54481_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=5429
[junit4] 2> 3374524 INFO (OverseerCollectionConfigSetProcessor-98896958294982660-127.0.0.1:59675_-n_0000000000) [n:127.0.0.1:59675_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3375177 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-3-001 of type TLOG
[junit4] 2> 3375178 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3375198 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@61c1d54a{/,null,AVAILABLE}
[junit4] 2> 3375198 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@d946d3f{HTTP/1.1,[http/1.1]}{127.0.0.1:49205}
[junit4] 2> 3375198 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.e.j.s.Server Started @3396638ms
[junit4] 2> 3375198 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/tempDir-001/jetty3, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=60243, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-001/shard-3-001/cores}
[junit4] 2> 3375198 ERROR (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3375216 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 3375216 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3375216 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 3375216 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-26T19:57:46.174Z
[junit4] 2> 3375457 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3375457 INFO (TEST-HttpPartitionTest.test-seed#[4D0F748AACCA44B0]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_4D0F748AACCA44B0-00
[...truncated too long message...]
.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 10661078 INFO (zkCallback-4667-thread-2-processing-n:127.0.0.1:49106_solr) [n:127.0.0.1:49106_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:49106_solr
[junit4] 2> 10661097 INFO (jetty-closer-4643-thread-5) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7fff2926{/solr,null,UNAVAILABLE}
[junit4] 2> 10661194 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f5ab19a7e000b, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 10661195 INFO (zkCallback-4674-thread-2-processing-n:127.0.0.1:51315_solr) [n:127.0.0.1:51315_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:51315_solr
[junit4] 2> 10661238 INFO (coreCloseExecutor-10138-thread-1) [n:127.0.0.1:41606_solr c:facetcollection s:shard1 r:core_node2 x:facetcollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.facetcollection.shard1.leader, tag=1482760307
[junit4] 2> 10661241 INFO (coreCloseExecutor-10139-thread-1) [n:127.0.0.1:43360_solr c:facetcollection s:shard3 r:core_node12 x:facetcollection_shard3_replica_n11] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.facetcollection.shard3.leader, tag=496671074
[junit4] 2> 10661241 INFO (coreCloseExecutor-10137-thread-1) [n:127.0.0.1:34403_solr c:facetcollection s:shard2 r:core_node8 x:facetcollection_shard2_replica_n7] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.facetcollection.shard2.leader, tag=531754703
[junit4] 2> 10661241 INFO (coreCloseExecutor-10136-thread-1) [n:127.0.0.1:51315_solr c:facetcollection s:shard2 r:core_node6 x:facetcollection_shard2_replica_n5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.facetcollection.shard2.leader, tag=616504417
[junit4] 2> 10661410 INFO (jetty-closer-4643-thread-6) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@14b5e0d1{/solr,null,UNAVAILABLE}
[junit4] 2> 10661477 INFO (jetty-closer-4643-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4e938237{/solr,null,UNAVAILABLE}
[junit4] 2> 10661477 INFO (jetty-closer-4643-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@567c3434{/solr,null,UNAVAILABLE}
[junit4] 2> 10662627 INFO (jetty-closer-4643-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6229bb0c{/solr,null,UNAVAILABLE}
[junit4] 2> 10662973 INFO (jetty-closer-4643-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@45ade1de{/solr,null,UNAVAILABLE}
[junit4] 2> 10662989 ERROR (SUITE-BlockJoinFacetDistribTest-seed#[4D0F748AACCA44B0]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 10662989 INFO (SUITE-BlockJoinFacetDistribTest-seed#[4D0F748AACCA44B0]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46876 46876
[junit4] 2> 10663109 INFO (Thread-8278) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46876 46876
[junit4] 2> 10663110 WARN (Thread-8278) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/aliases.json
[junit4] 2> 8 /solr/clusterprops.json
[junit4] 2> 6 /solr/security.json
[junit4] 2> 6 /solr/configs/solrCloudCollectionConfig
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 91 /solr/collections/facetcollection/state.json
[junit4] 2> 8 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/live_nodes
[junit4] 2> 8 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.search.join.BlockJoinFacetDistribTest_4D0F748AACCA44B0-001
[junit4] 2> Oct 26, 2017 9:59:11 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1838, maxMBSortInHeap=6.924484135782889, sim=RandomSimilarity(queryNorm=false): {}, locale=hi-IN, timezone=Europe/Mariehamn
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=248313664,total=526385152
[junit4] 2> NOTE: All tests run in this JVM: [TestJavabinTupleStreamParser, TestNumericTerms32, OverseerRolesTest, CloudMLTQParserTest, BadCopyFieldTest, TestHighlightDedupGrouping, MultiTermTest, QueryParsingTest, TestReplicationHandler, ResponseLogComponentTest, MoveReplicaHDFSFailoverTest, DateFieldTest, BaseCdcrDistributedZkTest, TestSolrCloudWithKerberosAlt, TestAuthorizationFramework, HttpTriggerListenerTest, TestIBSimilarityFactory, SolrJmxReporterCloudTest, SliceStateTest, HdfsNNFailoverTest, DeleteReplicaTest, GraphQueryTest, SolrCoreTest, SolrCloudReportersTest, TestSolrCLIRunExample, SyncSliceTest, DistributedQueryComponentCustomSortTest, TestRestManager, TestUtils, TestSchemaVersionResource, DistributedQueueTest, HLLSerializationTest, VersionInfoTest, ResourceLoaderTest, TestNoOpRegenerator, TestLuceneIndexBackCompat, CollectionStateFormat2Test, DirectUpdateHandlerOptimizeTest, CoreAdminCreateDiscoverTest, BitVectorTest, TestManagedStopFilterFactory, CursorPagingTest, TestSolrDeletionPolicy1, TestPrepRecovery, TestCrossCoreJoin, TestHashQParserPlugin, TestBinaryField, TestMaxScoreQueryParser, DocValuesTest, RAMDirectoryFactoryTest, TestOrdValues, NoCacheHeaderTest, BasicAuthIntegrationTest, CurrencyFieldTypeTest, TestLockTree, AddBlockUpdateTest, TestRecovery, FileUtilsTest, IndexSchemaRuntimeFieldTest, TestMaxTokenLenTokenizer, TestEmbeddedSolrServerSchemaAPI, TestAuthenticationFramework, TestSchemaNameResource, TestDocBasedVersionConstraints, TestTrieFacet, ShardSplitTest, TestSweetSpotSimilarityFactory, TestReversedWildcardFilterFactory, TestAnalyzeInfixSuggestions, AutoAddReplicasIntegrationTest, TestHalfAndHalfDocValues, TestCodecSupport, LeaderInitiatedRecoveryOnShardRestartTest, MinimalSchemaTest, DistributedMLTComponentTest, TermVectorComponentTest, TestExactSharedStatsCache, BlockDirectoryTest, HdfsRestartWhileUpdatingTest, TestRawTransformer, TestComplexPhraseLeadingWildcard, FacetPivotSmallTest, TestSolrFieldCacheBean, DistribCursorPagingTest, TestLeaderElectionZkExpiry, SimpleFacetsTest, TestSolrIndexConfig, TestReplicaProperties, TestCloudManagedSchema, TestCoreAdminApis, LeaderElectionIntegrationTest, TestMergePolicyConfig, TestJoin, TestFoldingMultitermQuery, TestPHPSerializedResponseWriter, EnumFieldTest, TriLevelCompositeIdRoutingTest, PeerSyncReplicationTest, TestFieldResource, SuggestComponentTest, ClassificationUpdateProcessorTest, TestSha256AuthenticationProvider, TestDeleteCollectionOnDownNodes, SolrCoreMetricManagerTest, JsonLoaderTest, OverriddenZkACLAndCredentialsProvidersTest, QueryResultKeyTest, SolrCLIZkUtilsTest, DateMathParserTest, BasicFunctionalityTest, CleanupOldIndexTest, StatelessScriptUpdateProcessorFactoryTest, TestCorePropertiesReload, TestCSVLoader, SpellCheckCollatorTest, ZkControllerTest, BasicDistributedZkTest, SearchRateTriggerTest, SearchHandlerTest, TestFreeTextSuggestions, CloudExitableDirectoryReaderTest, CollectionsAPIAsyncDistributedZkTest, CustomCollectionTest, DeleteStatusTest, DocValuesNotIndexedTest, LeaderFailureAfterFreshStartTest, LeaderInitiatedRecoveryOnCommitTest, NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerStatusTest, RecoveryAfterSoftCommitTest, SaslZkACLProviderTest, SolrXmlInZkTest, TestCloudDeleteByQuery, TestCloudPseudoReturnFields, TestClusterProperties, TestCollectionAPI, TestConfigSetsAPIExclusivity, TestDistribDocBasedVersion, TestExclusionRuleCollectionAccess, TestHdfsCloudBackupRestore, TestRebalanceLeaders, TestSSLRandomization, TestSolrCloudWithSecureImpersonation, TestStressInPlaceUpdates, HdfsBasicDistributedZk2Test, HdfsChaosMonkeyNothingIsSafeTest, HdfsCollectionsAPIDistributedZkTest, HdfsRecoveryZkTest, RuleEngineTest, RulesTest, TestConfigSets, TestImplicitCoreProperties, TestInfoStreamLogging, TestShardHandlerFactory, TestSimpleTextCodec, TestSolrConfigHandler, TestSolrConfigHandlerCloud, DistributedFacetPivotLongTailTest, DistributedFacetPivotSmallTest, DistributedQueryComponentOptimizationTest, ShufflingReplicaListTransformerTest, SpatialHeatmapFacetsTest, TestHttpShardHandlerFactory, JavabinLoaderTest, TestSlowCompositeReaderWrapper, WrapperMergePolicyFactoryTest, CheckHdfsIndexTest, TestMacroExpander, SmileWriterTest, TestBinaryResponseWriter, TestExportWriter, TestSubQueryTransformer, TestManagedResourceStorage, TestDynamicFieldCollectionResource, TestSerializedLuceneMatchVersion, DateRangeFieldTest, SpatialRPTFieldTypeTest, TestManagedSchema, TestManagedSchemaThreadSafety, TestPointFields, TestInitQParser, TestPayloadCheckQParserPlugin, TestRandomCollapseQParserPlugin, TestMinMaxOnMultiValuedField, BlockJoinFacetDistribTest]
[junit4] Completed [700/744 (6!)] on J0 in 42.69s, 1 test, 1 error <<< FAILURES!
[...truncated 47385 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 193 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/193/
8 tests failed.
FAILED: org.apache.solr.cloud.DeleteReplicaTest.deleteReplicaAndVerifyDirectoryCleanup
Error Message:
No registered leader was found after waiting for 4000ms , collection: deletereplica_test slice: shard1 saw state=DocCollection(deletereplica_test//collections/deletereplica_test/state.json/9)={ "pullReplicas":"0", "replicationFactor":"2", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{"core_node3":{ "core":"deletereplica_test_shard1_replica_n1", "base_url":"https://127.0.0.1:58341/solr", "node_name":"127.0.0.1:58341_solr", "state":"active", "type":"NRT", "leader":"true"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"2", "tlogReplicas":"0"} with live_nodes=[127.0.0.1:50213_solr, 127.0.0.1:58341_solr, 127.0.0.1:55321_solr, 127.0.0.1:50473_solr]
Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 4000ms , collection: deletereplica_test slice: shard1 saw state=DocCollection(deletereplica_test//collections/deletereplica_test/state.json/9)={
"pullReplicas":"0",
"replicationFactor":"2",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{"core_node3":{
"core":"deletereplica_test_shard1_replica_n1",
"base_url":"https://127.0.0.1:58341/solr",
"node_name":"127.0.0.1:58341_solr",
"state":"active",
"type":"NRT",
"leader":"true"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"2",
"tlogReplicas":"0"} with live_nodes=[127.0.0.1:50213_solr, 127.0.0.1:58341_solr, 127.0.0.1:55321_solr, 127.0.0.1:50473_solr]
at __randomizedtesting.SeedInfo.seed([2BF42D8AD9FD7CA4:5B14A9305DB2D5AF]:0)
at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:816)
at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:800)
at org.apache.solr.cloud.DeleteReplicaTest.deleteReplicaAndVerifyDirectoryCleanup(DeleteReplicaTest.java:93)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestAuthenticationFramework.testBasics
Error Message:
Error from server at https://127.0.0.1:59320/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason: <pre> Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html>
Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:59320/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason:
<pre> Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>
at __randomizedtesting.SeedInfo.seed([2BF42D8AD9FD7CA4:162C83A6E11322D4]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:541)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:998)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
at org.apache.solr.cloud.TestAuthenticationFramework.collectionCreateSearchDeleteTwice(TestAuthenticationFramework.java:126)
at org.apache.solr.cloud.TestAuthenticationFramework.testBasics(TestAuthenticationFramework.java:74)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:59320/solr/testcollection_shard1_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n2/update. Reason:
<pre> Can not find: /solr/testcollection_shard1_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:590)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:518)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
FAILED: org.apache.solr.cloud.TestCloudPseudoReturnFields.testAugmentersGlobsExplicitAndScoreOhMyRTG
Error Message:
Error from server at https://127.0.0.1:51505/solr/org.apache.solr.cloud.TestCloudPseudoReturnFields_collection: no servers hosting shard: shard2
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:51505/solr/org.apache.solr.cloud.TestCloudPseudoReturnFields_collection: no servers hosting shard: shard2
at __randomizedtesting.SeedInfo.seed([2BF42D8AD9FD7CA4:76C9D67605608E]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.client.solrj.SolrClient.getById(SolrClient.java:1173)
at org.apache.solr.client.solrj.SolrClient.getById(SolrClient.java:1090)
at org.apache.solr.client.solrj.SolrClient.getById(SolrClient.java:1110)
at org.apache.solr.cloud.TestCloudPseudoReturnFields.testAugmentersGlobsExplicitAndScoreOhMyRTG(TestCloudPseudoReturnFields.java:779)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestCollectionAPI.test
Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:43790
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:43790
at __randomizedtesting.SeedInfo.seed([2BF42D8AD9FD7CA4:A3A012507701115C]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:637)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.TestCollectionAPI.testClusterStateMigration(TestCollectionAPI.java:645)
at org.apache.solr.cloud.TestCollectionAPI.test(TestCollectionAPI.java:94)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
at sun.security.ssl.InputRecord.read(InputRecord.java:503)
at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:940)
at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
... 52 more
FAILED: org.apache.solr.cloud.TestSegmentSorting.testAtomicUpdateOfSegmentSortField
Error Message:
Error from server at http://127.0.0.1:35191/solr/testAtomicUpdateOfSegmentSortField_shard1_replica_n1: Cannot talk to ZooKeeper - Updates are disabled.
Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:35191/solr/testAtomicUpdateOfSegmentSortField_shard1_replica_n1: Cannot talk to ZooKeeper - Updates are disabled.
at __randomizedtesting.SeedInfo.seed([2BF42D8AD9FD7CA4:CAE60A8F0FB8E71C]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:541)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:998)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
at org.apache.solr.cloud.TestSegmentSorting.testAtomicUpdateOfSegmentSortField(TestSegmentSorting.java:140)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:47)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:35191/solr/testAtomicUpdateOfSegmentSortField_shard1_replica_n1: Cannot talk to ZooKeeper - Updates are disabled.
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:518)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
FAILED: org.apache.solr.handler.TestSolrConfigHandlerCloud.test
Error Message:
{ "responseHeader":{ "status":500, "QTime":32476}, "errorMessages":["4 out of 5 the property overlay to be of version 3 within 30 seconds! Failed cores: [https://127.0.0.1:56023/collection1_shard2_replica_n41/, https://127.0.0.1:54547/collection1_shard2_replica_n45/, https://127.0.0.1:52772/collection1_shard1_replica_n43/, https://127.0.0.1:50133/collection1_shard1_replica_n47/]\n"], "WARNING":"This response format is experimental. It is likely to change in the future.", "error":{ "metadata":[ "error-class","org.apache.solr.common.SolrException", "root-error-class","org.apache.solr.common.SolrException"], "msg":"4 out of 5 the property overlay to be of version 3 within 30 seconds! Failed cores: [https://127.0.0.1:56023/collection1_shard2_replica_n41/, https://127.0.0.1:54547/collection1_shard2_replica_n45/, https://127.0.0.1:52772/collection1_shard1_replica_n43/, https://127.0.0.1:50133/collection1_shard1_replica_n47/]", "trace":"org.apache.solr.common.SolrException: 4 out of 5 the property overlay to be of version 3 within 30 seconds! Failed cores: [https://127.0.0.1:56023/collection1_shard2_replica_n41/, https://127.0.0.1:54547/collection1_shard2_replica_n45/, https://127.0.0.1:52772/collection1_shard1_replica_n43/, https://127.0.0.1:50133/collection1_shard1_replica_n47/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:772)\n\tat org.apache.solr.handler.SolrConfigHandler.access$400(SolrConfigHandler.java:102)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:503)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:346)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.access$100(SolrConfigHandler.java:159)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2484)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:710)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:516)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:534)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)\n\tat java.lang.Thread.run(Thread.java:748)\n", "code":500}}
Stack Trace:
java.lang.AssertionError: {
"responseHeader":{
"status":500,
"QTime":32476},
"errorMessages":["4 out of 5 the property overlay to be of version 3 within 30 seconds! Failed cores: [https://127.0.0.1:56023/collection1_shard2_replica_n41/, https://127.0.0.1:54547/collection1_shard2_replica_n45/, https://127.0.0.1:52772/collection1_shard1_replica_n43/, https://127.0.0.1:50133/collection1_shard1_replica_n47/]\n"],
"WARNING":"This response format is experimental. It is likely to change in the future.",
"error":{
"metadata":[
"error-class","org.apache.solr.common.SolrException",
"root-error-class","org.apache.solr.common.SolrException"],
"msg":"4 out of 5 the property overlay to be of version 3 within 30 seconds! Failed cores: [https://127.0.0.1:56023/collection1_shard2_replica_n41/, https://127.0.0.1:54547/collection1_shard2_replica_n45/, https://127.0.0.1:52772/collection1_shard1_replica_n43/, https://127.0.0.1:50133/collection1_shard1_replica_n47/]",
"trace":"org.apache.solr.common.SolrException: 4 out of 5 the property overlay to be of version 3 within 30 seconds! Failed cores: [https://127.0.0.1:56023/collection1_shard2_replica_n41/, https://127.0.0.1:54547/collection1_shard2_replica_n45/, https://127.0.0.1:52772/collection1_shard1_replica_n43/, https://127.0.0.1:50133/collection1_shard1_replica_n47/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:772)\n\tat org.apache.solr.handler.SolrConfigHandler.access$400(SolrConfigHandler.java:102)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:503)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:346)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.access$100(SolrConfigHandler.java:159)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2484)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:710)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:516)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:534)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)\n\tat java.lang.Thread.run(Thread.java:748)\n",
"code":500}}
at __randomizedtesting.SeedInfo.seed([2BF42D8AD9FD7CA4:A3A012507701115C]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.core.TestSolrConfigHandler.runConfigCommand(TestSolrConfigHandler.java:185)
at org.apache.solr.core.TestSolrConfigHandler.reqhandlertests(TestSolrConfigHandler.java:253)
at org.apache.solr.handler.TestSolrConfigHandlerCloud.testReqHandlerAPIs(TestSolrConfigHandlerCloud.java:97)
at org.apache.solr.handler.TestSolrConfigHandlerCloud.test(TestSolrConfigHandlerCloud.java:67)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.update.SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds
Error Message:
softCommit: did not find a single commit
Stack Trace:
java.lang.AssertionError: softCommit: did not find a single commit
at __randomizedtesting.SeedInfo.seed([2BF42D8AD9FD7CA4:BE5A8BDD3BEF0ED9]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNotNull(Assert.java:526)
at org.apache.solr.update.SoftAutoCommitTest.assertRapidMultiCommitQueues(SoftAutoCommitTest.java:494)
at org.apache.solr.update.SoftAutoCommitTest.doTestSoftAndHardCommitMaxTimeRapidAdds(SoftAutoCommitTest.java:459)
at org.apache.solr.update.SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds(SoftAutoCommitTest.java:411)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.client.solrj.impl.CloudSolrClientTest.preferLocalShardsTest
Error Message:
Could not load collection from ZK: localShardsTestColl
Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: localShardsTestColl
at __randomizedtesting.SeedInfo.seed([D5E1F801E0D45786:292C60B81E03C07C]:0)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
at org.apache.solr.client.solrj.impl.CloudSolrClient.getDocCollection(CloudSolrClient.java:1188)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:832)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.client.solrj.impl.CloudSolrClientTest.queryWithPreferLocalShards(CloudSolrClientTest.java:456)
at org.apache.solr.client.solrj.impl.CloudSolrClientTest.preferLocalShardsTest(CloudSolrClientTest.java:437)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:110)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/localShardsTestColl/state.json
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
... 48 more
Build Log:
[...truncated 11766 lines...]
[junit4] Suite: org.apache.solr.cloud.DeleteReplicaTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/init-core-data-001
[junit4] 2> 733275 WARN (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=643 numCloses=643
[junit4] 2> 733275 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 733277 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 733292 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 733293 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001
[junit4] 2> 733293 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 733301 INFO (Thread-192) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 733302 INFO (Thread-192) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 733432 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:36541
[junit4] 2> 733463 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
[junit4] 2> 733472 ERROR (Thread-192) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 735213 INFO (jetty-launcher-154-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 735214 INFO (jetty-launcher-154-thread-3) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 735220 INFO (jetty-launcher-154-thread-4) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 735224 INFO (jetty-launcher-154-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@d87a547{/solr,null,AVAILABLE}
[junit4] 2> 735225 INFO (jetty-launcher-154-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5cf2639a{/solr,null,AVAILABLE}
[junit4] 2> 735243 INFO (jetty-launcher-154-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@8ce55ac{/solr,null,AVAILABLE}
[junit4] 2> 735302 INFO (jetty-launcher-154-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 735332 INFO (jetty-launcher-154-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@167b9c6b{SSL,[ssl, http/1.1]}{127.0.0.1:50213}
[junit4] 2> 735332 INFO (jetty-launcher-154-thread-3) [ ] o.e.j.s.Server Started @760069ms
[junit4] 2> 735332 INFO (jetty-launcher-154-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50213}
[junit4] 2> 735355 INFO (jetty-launcher-154-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@23fd37e7{SSL,[ssl, http/1.1]}{127.0.0.1:58341}
[junit4] 2> 735355 INFO (jetty-launcher-154-thread-4) [ ] o.e.j.s.Server Started @760092ms
[junit4] 2> 735355 INFO (jetty-launcher-154-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58341}
[junit4] 2> 735356 ERROR (jetty-launcher-154-thread-4) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-25T23:49:57.055Z
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1a1abeb8{SSL,[ssl, http/1.1]}{127.0.0.1:50473}
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-2) [ ] o.e.j.s.Server Started @760093ms
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50473}
[junit4] 2> 735356 ERROR (jetty-launcher-154-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 735356 INFO (jetty-launcher-154-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 735357 INFO (jetty-launcher-154-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 735357 INFO (jetty-launcher-154-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-25T23:49:57.056Z
[junit4] 2> 735375 ERROR (jetty-launcher-154-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 735375 INFO (jetty-launcher-154-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 735375 INFO (jetty-launcher-154-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 735375 INFO (jetty-launcher-154-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 735375 INFO (jetty-launcher-154-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-25T23:49:57.074Z
[junit4] 2> 735376 INFO (jetty-launcher-154-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@a601c96{/solr,null,AVAILABLE}
[junit4] 2> 735455 INFO (jetty-launcher-154-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6b090788{SSL,[ssl, http/1.1]}{127.0.0.1:55321}
[junit4] 2> 735455 INFO (jetty-launcher-154-thread-1) [ ] o.e.j.s.Server Started @760192ms
[junit4] 2> 735455 INFO (jetty-launcher-154-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=55321}
[junit4] 2> 735455 ERROR (jetty-launcher-154-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 735455 INFO (jetty-launcher-154-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 735455 INFO (jetty-launcher-154-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 735455 INFO (jetty-launcher-154-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 735455 INFO (jetty-launcher-154-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-25T23:49:57.154Z
[junit4] 2> 735553 INFO (jetty-launcher-154-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 735555 INFO (jetty-launcher-154-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 735584 INFO (jetty-launcher-154-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 735591 INFO (jetty-launcher-154-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 735594 INFO (jetty-launcher-154-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36541/solr
[junit4] 2> 735715 INFO (jetty-launcher-154-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36541/solr
[junit4] 2> 735751 INFO (jetty-launcher-154-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36541/solr
[junit4] 2> 735819 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f55f1415a0005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 736091 INFO (jetty-launcher-154-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36541/solr
[junit4] 2> 736227 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f55f1415a0007, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 736491 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 736492 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:50473_solr
[junit4] 2> 736499 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.c.Overseer Overseer (id=98892211497336840-127.0.0.1:50473_solr-n_0000000000) starting
[junit4] 2> 736539 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50473_solr
[junit4] 2> 736552 INFO (jetty-launcher-154-thread-3) [n:127.0.0.1:50213_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 736552 INFO (jetty-launcher-154-thread-3) [n:127.0.0.1:50213_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50213_solr
[junit4] 2> 736691 INFO (zkCallback-174-thread-1-processing-n:127.0.0.1:50213_solr) [n:127.0.0.1:50213_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 736701 INFO (OverseerStateUpdate-98892211497336840-127.0.0.1:50473_solr-n_0000000000) [n:127.0.0.1:50473_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 736994 INFO (zkCallback-177-thread-1-processing-n:127.0.0.1:58341_solr) [n:127.0.0.1:58341_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 736997 INFO (jetty-launcher-154-thread-4) [n:127.0.0.1:58341_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 737001 INFO (jetty-launcher-154-thread-4) [n:127.0.0.1:58341_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58341_solr
[junit4] 2> 737003 INFO (zkCallback-174-thread-1-processing-n:127.0.0.1:50213_solr) [n:127.0.0.1:50213_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 737007 INFO (zkCallback-177-thread-1-processing-n:127.0.0.1:58341_solr) [n:127.0.0.1:58341_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 737012 INFO (zkCallback-172-thread-1-processing-n:127.0.0.1:50473_solr) [n:127.0.0.1:50473_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 738052 INFO (jetty-launcher-154-thread-3) [n:127.0.0.1:50213_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50213.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 738143 INFO (jetty-launcher-154-thread-3) [n:127.0.0.1:50213_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50213.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 738143 INFO (jetty-launcher-154-thread-3) [n:127.0.0.1:50213_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50213.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 738179 INFO (jetty-launcher-154-thread-3) [n:127.0.0.1:50213_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node3/.
[junit4] 2> 739631 INFO (jetty-launcher-154-thread-1) [n:127.0.0.1:55321_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 739632 INFO (jetty-launcher-154-thread-1) [n:127.0.0.1:55321_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 739643 INFO (jetty-launcher-154-thread-1) [n:127.0.0.1:55321_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55321_solr
[junit4] 2> 739644 INFO (zkCallback-172-thread-1-processing-n:127.0.0.1:50473_solr) [n:127.0.0.1:50473_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 739645 INFO (zkCallback-174-thread-1-processing-n:127.0.0.1:50213_solr) [n:127.0.0.1:50213_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 739651 INFO (zkCallback-177-thread-1-processing-n:127.0.0.1:58341_solr) [n:127.0.0.1:58341_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 739715 INFO (zkCallback-176-thread-1-processing-n:127.0.0.1:55321_solr) [n:127.0.0.1:55321_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 743877 INFO (jetty-launcher-154-thread-4) [n:127.0.0.1:58341_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58341.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744111 INFO (jetty-launcher-154-thread-4) [n:127.0.0.1:58341_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58341.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744119 INFO (jetty-launcher-154-thread-4) [n:127.0.0.1:58341_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58341.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744120 INFO (jetty-launcher-154-thread-4) [n:127.0.0.1:58341_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node4/.
[junit4] 2> 744247 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50473.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744278 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50473.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744279 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50473.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744328 INFO (jetty-launcher-154-thread-2) [n:127.0.0.1:50473_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node2/.
[junit4] 2> 744518 INFO (jetty-launcher-154-thread-1) [n:127.0.0.1:55321_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55321.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744599 INFO (jetty-launcher-154-thread-1) [n:127.0.0.1:55321_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55321.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744600 INFO (jetty-launcher-154-thread-1) [n:127.0.0.1:55321_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55321.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 744601 INFO (jetty-launcher-154-thread-1) [n:127.0.0.1:55321_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node1/.
[junit4] 2> 745763 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 745764 INFO (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36541/solr ready
[junit4] 2> 746042 INFO (TEST-DeleteReplicaTest.deleteReplicaAndVerifyDirectoryCleanup-seed#[2BF42D8AD9FD7CA4]) [ ] o.a.s.SolrTestCaseJ4 ###Starting deleteReplicaAndVerifyDirectoryCleanup
[junit4] 2> 746449 INFO (qtp710352259-1234) [n:127.0.0.1:50213_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&name=deletereplica_test&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 746531 INFO (OverseerThreadFactory-432-thread-1-processing-n:127.0.0.1:50473_solr) [n:127.0.0.1:50473_solr ] o.a.s.c.CreateCollectionCmd Create collection deletereplica_test
[junit4] 2> 746682 INFO (OverseerStateUpdate-98892211497336840-127.0.0.1:50473_solr-n_0000000000) [n:127.0.0.1:50473_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"deletereplica_test",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"deletereplica_test_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:58341/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 746713 INFO (OverseerStateUpdate-98892211497336840-127.0.0.1:50473_solr-n_0000000000) [n:127.0.0.1:50473_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"deletereplica_test",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"deletereplica_test_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:50213/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 749121 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=deletereplica_test_shard1_replica_n2&action=CREATE&numShards=1&collection=deletereplica_test&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 749122 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 749173 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 758284 WARN (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker-SendThread(127.0.0.1:36541)) [ ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 11735ms for sessionid 0x15f55f1415a000e
[junit4] 2> 758421 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=deletereplica_test_shard1_replica_n1&action=CREATE&numShards=1&collection=deletereplica_test&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 758423 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 758429 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.s.IndexSchema [deletereplica_test_shard1_replica_n2] Schema name=minimal
[junit4] 2> 758636 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 758636 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'deletereplica_test_shard1_replica_n2' using configuration from collection deletereplica_test, trusted=true
[junit4] 2> 758637 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_50213.solr.core.deletereplica_test.shard1.replica_n2' (registry 'solr.core.deletereplica_test.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 758642 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 758642 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.c.SolrCore [[deletereplica_test_shard1_replica_n2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node3/deletereplica_test_shard1_replica_n2], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node3/./deletereplica_test_shard1_replica_n2/data/]
[junit4] 2> 758643 WARN (zkCallback-185-thread-2) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5b88fda5 name: ZooKeeperConnection Watcher:127.0.0.1:36541/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 758643 WARN (zkCallback-185-thread-2) [ ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 758648 INFO (zkCallback-174-thread-1-processing-n:127.0.0.1:50213_solr) [n:127.0.0.1:50213_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deletereplica_test/state.json] for collection [deletereplica_test] has occurred - updating... (live nodes size: [4])
[junit4] 2> 758707 INFO (zkCallback-177-thread-1-processing-n:127.0.0.1:58341_solr) [n:127.0.0.1:58341_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deletereplica_test/state.json] for collection [deletereplica_test] has occurred - updating... (live nodes size: [4])
[junit4] 2> 758707 INFO (zkCallback-177-thread-2-processing-n:127.0.0.1:58341_solr) [n:127.0.0.1:58341_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deletereplica_test/state.json] for collection [deletereplica_test] has occurred - updating... (live nodes size: [4])
[junit4] 2> 759403 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 759403 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 759434 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 759434 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 759633 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@38c628b4[deletereplica_test_shard1_replica_n2] main]
[junit4] 2> 759635 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 759636 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 759637 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 759876 INFO (searcherExecutor-441-thread-1-processing-n:127.0.0.1:50213_solr x:deletereplica_test_shard1_replica_n2 s:shard1 c:deletereplica_test r:core_node4) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.c.SolrCore [deletereplica_test_shard1_replica_n2] Registered new searcher Searcher@38c628b4[deletereplica_test_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 759899 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr c:deletereplica_test s:shard1 r:core_node3 x:deletereplica_test_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 759935 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582275411715293184
[junit4] 2> 759956 INFO (qtp710352259-1232) [n:127.0.0.1:50213_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 759995 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr c:deletereplica_test s:shard1 r:core_node3 x:deletereplica_test_shard1_replica_n1] o.a.s.s.IndexSchema [deletereplica_test_shard1_replica_n1] Schema name=minimal
[junit4] 2> 759997 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr c:deletereplica_test s:shard1 r:core_node3 x:deletereplica_test_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 759997 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr c:deletereplica_test s:shard1 r:core_node3 x:deletereplica_test_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'deletereplica_test_shard1_replica_n1' using configuration from collection deletereplica_test, trusted=true
[junit4] 2> 759998 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr c:deletereplica_test s:shard1 r:core_node3 x:deletereplica_test_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58341.solr.core.deletereplica_test.shard1.replica_n1' (registry 'solr.core.deletereplica_test.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ca7b2
[junit4] 2> 759998 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr c:deletereplica_test s:shard1 r:core_node3 x:deletereplica_test_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 759998 INFO (qtp1390311667-1225) [n:127.0.0.1:58341_solr c:deletereplica_test s:shard1 r:core_node3 x:deletereplica_test_shard1_replica_n1] o.a.s.c.SolrCore [[deletereplica_test_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node4/deletereplica_test_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteReplicaTest_2BF42D8AD9FD7CA4-001/tempDir-001/node4/./deletereplica_test_shard1_replica_n1/data/]
[junit4] 2> 760059 INFO (zkCallback-174-thread-1-processing-n:127.0.0.1:50213_solr) [n:127.0.0.1:50213_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deletereplica_test/state.json] for collection [deletereplica_test] has occurred - updating... (live nodes size: [4])
[junit4] 2> 760059 INFO (zkCallback-177-thread-2-processing-n:127.0.0.1:58341_solr) [n:127.0.0.1:58341_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deletereplica_test/state.json] for collection [deletereplica_test] has occurred - updating... (live nodes size: [4])
[junit4] 2> 760059 INFO (zkCallback-177-thread-1-processing-n:127.0.0.1:58341_solr) [n:127.0.0.1:58341_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/deletereplica_test/state.json] for collection [deletereplica_test] has occurred - updating... (live nodes size: [4])
[junit4] 2> 760287 WARN (SUITE-DeleteReplicaTest-seed#[2BF42D8AD9FD7CA4]-worker-SendThread(127.0.0.1:36541)) [ ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15f55f1415a000e has expired
[junit4] 2> 760323 WARN (zkCallback-185-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5b88fda5 name: ZooKeeperConnection Watcher:127.0.0.1:36541/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 760323 WARN (zkCallback-185-thread-1) [ ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 760323 WARN (zkCallback-185-thread-1) [ ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 760439 INFO (zkCallback-185-thread-1) [ ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 760656 INFO (zkCallback-185-thread-1) [ ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper
[ju
[...truncated too long message...]
re.2nd_collection.shard1.replica_n1, tag=1848148207
[junit4] 2> 463140 INFO (coreCloseExecutor-301-thread-6) [n:127.0.0.1:39390_solr c:2nd_collection s:shard1 r:core_node3 x:2nd_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@570c1239: rootName = solr_39390, domain = solr.core.2nd_collection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.2nd_collection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@3c5e08cb
[junit4] 2> 463147 INFO (coreCloseExecutor-303-thread-3) [n:127.0.0.1:59769_solr c:localShardsTestColl s:shard1 r:core_node2 x:localShardsTestColl_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard1.leader, tag=823262030
[junit4] 2> 463172 INFO (coreCloseExecutor-301-thread-7) [n:127.0.0.1:39390_solr c:multicollection1 s:shard2 r:core_node4 x:multicollection1_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.multicollection1.shard2.replica_n2, tag=1956061779
[junit4] 2> 463199 INFO (coreCloseExecutor-301-thread-7) [n:127.0.0.1:39390_solr c:multicollection1 s:shard2 r:core_node4 x:multicollection1_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@24acb18a: rootName = solr_39390, domain = solr.core.multicollection1.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.multicollection1.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@1f6619b8
[junit4] 2> 463344 INFO (coreCloseExecutor-301-thread-1) [n:127.0.0.1:39390_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=1666671325
[junit4] 2> 463345 INFO (coreCloseExecutor-301-thread-6) [n:127.0.0.1:39390_solr c:2nd_collection s:shard1 r:core_node3 x:2nd_collection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.2nd_collection.shard1.leader, tag=1848148207
[junit4] 2> 463345 INFO (coreCloseExecutor-301-thread-5) [n:127.0.0.1:39390_solr c:localShardsTestColl s:shard3 r:core_node18 x:localShardsTestColl_shard3_replica_n17] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard3.leader, tag=1216561606
[junit4] 2> 463345 INFO (coreCloseExecutor-301-thread-7) [n:127.0.0.1:39390_solr c:multicollection1 s:shard2 r:core_node4 x:multicollection1_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.multicollection1.shard2.leader, tag=1956061779
[junit4] 2> 463347 INFO (coreCloseExecutor-301-thread-4) [n:127.0.0.1:39390_solr c:localShardsTestColl s:shard1 r:core_node6 x:localShardsTestColl_shard1_replica_n5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard1.leader, tag=1638113748
[junit4] 2> 463367 INFO (coreCloseExecutor-302-thread-4) [n:127.0.0.1:60025_solr c:localShardsTestColl s:shard2 r:core_node10 x:localShardsTestColl_shard2_replica_n9] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard2.leader, tag=1686851154
[junit4] 2> 463367 INFO (coreCloseExecutor-302-thread-5) [n:127.0.0.1:60025_solr c:2nd_collection s:shard2 r:core_node4 x:2nd_collection_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.2nd_collection.shard2.replica_n2, tag=548992199
[junit4] 2> 463367 INFO (coreCloseExecutor-302-thread-5) [n:127.0.0.1:60025_solr c:2nd_collection s:shard2 r:core_node4 x:2nd_collection_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@807e8be: rootName = solr_60025, domain = solr.core.2nd_collection.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.2nd_collection.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@503125f7
[junit4] 2> 463383 INFO (coreCloseExecutor-302-thread-5) [n:127.0.0.1:60025_solr c:2nd_collection s:shard2 r:core_node4 x:2nd_collection_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.2nd_collection.shard2.leader, tag=548992199
[junit4] 2> 463595 INFO (coreCloseExecutor-302-thread-6) [n:127.0.0.1:60025_solr c:multicollection2 s:shard2 r:core_node4 x:multicollection2_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.multicollection2.shard2.replica_n2, tag=1028853355
[junit4] 2> 463595 INFO (coreCloseExecutor-302-thread-6) [n:127.0.0.1:60025_solr c:multicollection2 s:shard2 r:core_node4 x:multicollection2_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4b88932e: rootName = solr_60025, domain = solr.core.multicollection2.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.multicollection2.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@357c1301
[junit4] 2> 463635 INFO (coreCloseExecutor-302-thread-6) [n:127.0.0.1:60025_solr c:multicollection2 s:shard2 r:core_node4 x:multicollection2_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.multicollection2.shard2.leader, tag=1028853355
[junit4] 2> 464586 INFO (jetty-closer-97-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2a3505a1{/solr,null,UNAVAILABLE}
[junit4] 2> 464729 INFO (jetty-closer-97-thread-1) [ ] o.a.s.c.Overseer Overseer (id=98892935048396807-127.0.0.1:59769_solr-n_0000000000) closing
[junit4] 2> 464729 INFO (OverseerStateUpdate-98892935048396807-127.0.0.1:59769_solr-n_0000000000) [n:127.0.0.1:59769_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:59769_solr
[junit4] 2> 464990 WARN (OverseerAutoScalingTriggerThread-98892935048396807-127.0.0.1:59769_solr-n_0000000000) [n:127.0.0.1:59769_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 465247 INFO (jetty-closer-97-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@22af484b{/solr,null,UNAVAILABLE}
[junit4] 2> 465947 INFO (jetty-closer-97-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@47712441{/solr,null,UNAVAILABLE}
[junit4] 2> 465949 ERROR (SUITE-CloudSolrClientTest-seed#[D5E1F801E0D45786]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 465949 INFO (SUITE-CloudSolrClientTest-seed#[D5E1F801E0D45786]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:54220 54220
[junit4] 2> 466063 INFO (Thread-82) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:54220 54220
[junit4] 2> 466083 WARN (Thread-82) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/aliases.json
[junit4] 2> 3 /solr/security.json
[junit4] 2> 3 /solr/configs/conf
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 44 /solr/collections/localShardsTestColl/state.json
[junit4] 2> 33 /solr/collections/foo/state.json
[junit4] 2> 17 /solr/collections/collection1/state.json
[junit4] 2> 16 /solr/collections/2nd_collection/state.json
[junit4] 2> 16 /solr/collections/multicollection1/state.json
[junit4] 2> 15 /solr/collections/multicollection2/state.json
[junit4] 2> 14 /solr/collections/nemesis/state.json
[junit4] 2> 8 /solr/clusterprops.json
[junit4] 2> 8 /solr/clusterstate.json
[junit4] 2> 6 /solr/collections/overwrite/state.json
[junit4] 2> 2 /solr/collections/localShardsTestColl/leader_elect/shard3/election/98892935048396808-core_node16-n_0000000000
[junit4] 2> 2 /solr/collections/foo/leader_elect/shard1/election/98892935048396808-core_node4-n_0000000000
[junit4] 2> 2 /solr/collections/localShardsTestColl/leader_elect/shard1/election/98892935048396807-core_node2-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/live_nodes
[junit4] 2> 8 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.impl.CloudSolrClientTest_D5E1F801E0D45786-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {multiDefault=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=MockRandom), title_s=PostingsFormat(name=MockRandom), id=FST50, text=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{_version_=DocValuesFormat(name=Direct), multiDefault=DocValuesFormat(name=Lucene70), title_s=DocValuesFormat(name=Lucene70), intDefault=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=981, maxMBSortInHeap=6.251189272312671, sim=RandomSimilarity(queryNorm=true): {}, locale=ga-IE, timezone=Etc/GMT-8
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=147777368,total=442499072
[junit4] 2> NOTE: All tests run in this JVM: [GreaterThanEqualToEvaluatorTest, SolrExampleEmbeddedTest, UuidEvaluatorTest, SolrExampleStreamingBinaryTest, TestCollectionStateWatchers, CloudSolrClientTest]
[junit4] Completed [53/145 (1!)] on J1 in 142.96s, 15 tests, 1 error <<< FAILURES!
[...truncated 46589 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 192 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/192/
4 tests failed.
FAILED: org.apache.solr.cloud.TestTlogReplica.testKillTlogReplica
Error Message:
Error from server at https://127.0.0.1:43641/solr/tlog_replica_test_kill_tlog_replica_shard1_replica_t2: ClusterState says we are the leader (https://127.0.0.1:43641/solr/tlog_replica_test_kill_tlog_replica_shard1_replica_t2), but locally we don't think so. Request came from null
Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:43641/solr/tlog_replica_test_kill_tlog_replica_shard1_replica_t2: ClusterState says we are the leader (https://127.0.0.1:43641/solr/tlog_replica_test_kill_tlog_replica_shard1_replica_t2), but locally we don't think so. Request came from null
at __randomizedtesting.SeedInfo.seed([E99EB240D3F155EC:36AC76852252DAE1]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:541)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:998)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:927)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
at org.apache.solr.cloud.TestTlogReplica.testKillTlogReplica(TestTlogReplica.java:445)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:43641/solr/tlog_replica_test_kill_tlog_replica_shard1_replica_t2: ClusterState says we are the leader (https://127.0.0.1:43641/solr/tlog_replica_test_kill_tlog_replica_shard1_replica_t2), but locally we don't think so. Request came from null
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:518)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
FAILED: org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test
Error Message:
Expected 2 of 3 replicas to be active but only found 1; [core_node5:{"core":"c8n_1x3_lf_shard1_replica_t2","base_url":"http://127.0.0.1:45754/pjt","node_name":"127.0.0.1:45754_pjt","state":"active","type":"TLOG","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/17)={ "pullReplicas":"0", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{ "core_node4":{ "state":"down", "base_url":"http://127.0.0.1:42503/pjt", "core":"c8n_1x3_lf_shard1_replica_t1", "node_name":"127.0.0.1:42503_pjt", "type":"TLOG"}, "core_node5":{ "core":"c8n_1x3_lf_shard1_replica_t2", "base_url":"http://127.0.0.1:45754/pjt", "node_name":"127.0.0.1:45754_pjt", "state":"active", "type":"TLOG", "leader":"true"}, "core_node6":{ "core":"c8n_1x3_lf_shard1_replica_t3", "base_url":"http://127.0.0.1:40853/pjt", "node_name":"127.0.0.1:40853_pjt", "state":"down", "type":"TLOG"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"0", "tlogReplicas":"3"}
Stack Trace:
java.lang.AssertionError: Expected 2 of 3 replicas to be active but only found 1; [core_node5:{"core":"c8n_1x3_lf_shard1_replica_t2","base_url":"http://127.0.0.1:45754/pjt","node_name":"127.0.0.1:45754_pjt","state":"active","type":"TLOG","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/17)={
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{
"core_node4":{
"state":"down",
"base_url":"http://127.0.0.1:42503/pjt",
"core":"c8n_1x3_lf_shard1_replica_t1",
"node_name":"127.0.0.1:42503_pjt",
"type":"TLOG"},
"core_node5":{
"core":"c8n_1x3_lf_shard1_replica_t2",
"base_url":"http://127.0.0.1:45754/pjt",
"node_name":"127.0.0.1:45754_pjt",
"state":"active",
"type":"TLOG",
"leader":"true"},
"core_node6":{
"core":"c8n_1x3_lf_shard1_replica_t3",
"base_url":"http://127.0.0.1:40853/pjt",
"node_name":"127.0.0.1:40853_pjt",
"state":"down",
"type":"TLOG"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"0",
"tlogReplicas":"3"}
at __randomizedtesting.SeedInfo.seed([E99EB240D3F155EC:61CA8D9A7D0D3814]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:169)
at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:56)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestPrepRecovery.testLeaderUnloaded
Error Message:
Expected collection: testLeaderUnloaded to be live with 1 shard and 2 replicas null Live Nodes: [127.0.0.1:36908_solr, 127.0.0.1:41335_solr] Last available state: null
Stack Trace:
java.lang.AssertionError: Expected collection: testLeaderUnloaded to be live with 1 shard and 2 replicas
null
Live Nodes: [127.0.0.1:36908_solr, 127.0.0.1:41335_solr]
Last available state: null
at __randomizedtesting.SeedInfo.seed([E99EB240D3F155EC:BA39A1B62EDDBBEE]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
at org.apache.solr.cloud.TestPrepRecovery.testLeaderUnloaded(TestPrepRecovery.java:59)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestPrepRecovery.testLeaderNotResponding
Error Message:
Expected collection: testLeaderNotResponding to be live with 1 shard and 1 replicas null Live Nodes: [127.0.0.1:36908_solr, 127.0.0.1:41335_solr] Last available state: null
Stack Trace:
java.lang.AssertionError: Expected collection: testLeaderNotResponding to be live with 1 shard and 1 replicas
null
Live Nodes: [127.0.0.1:36908_solr, 127.0.0.1:41335_solr]
Last available state: null
at __randomizedtesting.SeedInfo.seed([E99EB240D3F155EC:B0C29A2794869553]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
at org.apache.solr.cloud.TestPrepRecovery.testLeaderNotResponding(TestPrepRecovery.java:93)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13440 lines...]
[junit4] Suite: org.apache.solr.cloud.LeaderFailoverAfterPartitionTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/init-core-data-001
[junit4] 2> 4792246 WARN (SUITE-LeaderFailoverAfterPartitionTest-seed#[E99EB240D3F155EC]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=7 numCloses=7
[junit4] 2> 4792246 INFO (SUITE-LeaderFailoverAfterPartitionTest-seed#[E99EB240D3F155EC]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 4792276 INFO (SUITE-LeaderFailoverAfterPartitionTest-seed#[E99EB240D3F155EC]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 4792276 INFO (SUITE-LeaderFailoverAfterPartitionTest-seed#[E99EB240D3F155EC]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 4792276 INFO (SUITE-LeaderFailoverAfterPartitionTest-seed#[E99EB240D3F155EC]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /pjt/
[junit4] 2> 4792277 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 4792313 INFO (Thread-3079) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 4792313 INFO (Thread-3079) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 4792357 ERROR (Thread-3079) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 4792406 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.ZkTestServer start zk server on port:46280
[junit4] 2> 4792582 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 4792584 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 4792617 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 4792618 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 4792619 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 4792619 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 4792620 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 4792620 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 4792639 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 4792640 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 4792640 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 4792654 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 4793291 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 4793365 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1f125ff{/pjt,null,AVAILABLE}
[junit4] 2> 4793365 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@361ed473{HTTP/1.1,[http/1.1]}{127.0.0.1:44291}
[junit4] 2> 4793365 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server Started @4806049ms
[junit4] 2> 4793365 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/tempDir-001/control/data, replicaType=NRT, hostContext=/pjt, hostPort=42503, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/control-001/cores}
[junit4] 2> 4793365 ERROR (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4793366 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 4793366 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4793366 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 4793366 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-24T12:09:46.134Z
[junit4] 2> 4793401 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 4793401 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/control-001/solr.xml
[junit4] 2> 4793404 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 4793404 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 4793418 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 4793454 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46280/solr
[junit4] 2> 4795489 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 4795489 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:42503_pjt
[junit4] 2> 4795490 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.c.Overseer Overseer (id=98883796015644676-127.0.0.1:42503_pjt-n_0000000000) starting
[junit4] 2> 4795517 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42503_pjt
[junit4] 2> 4795567 INFO (OverseerStateUpdate-98883796015644676-127.0.0.1:42503_pjt-n_0000000000) [n:127.0.0.1:42503_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4795772 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4795829 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4795829 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4795843 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:42503_pjt ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/control-001/cores
[junit4] 2> 4795929 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4795930 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46280/solr ready
[junit4] 2> 4795931 INFO (SocketProxy-Acceptor-42503) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=42696,localport=42503], receiveBufferSize:531000
[junit4] 2> 4795931 INFO (SocketProxy-Acceptor-42503) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=44291,localport=39882], receiveBufferSize=530904
[junit4] 2> 4795945 INFO (qtp1614536027-20976) [n:127.0.0.1:42503_pjt ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:42503_pjt&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4795950 INFO (OverseerThreadFactory-6955-thread-1-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 4796115 INFO (SocketProxy-Acceptor-42503) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=42702,localport=42503], receiveBufferSize:531000
[junit4] 2> 4796116 INFO (SocketProxy-Acceptor-42503) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=44291,localport=39888], receiveBufferSize=530904
[junit4] 2> 4796134 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 4796134 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 4796257 INFO (zkCallback-5067-thread-1-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4797224 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 4797278 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 4798132 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 4798189 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 4798190 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4798190 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 4798190 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 4798208 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=26, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 4798224 WARN (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 4798518 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 4798518 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 4798519 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 4798519 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 4798520 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=36.0888671875, floorSegmentMB=0.9384765625, forceMergeDeletesPctAllowed=2.6420731854266144, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 4798520 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@93a6a7[control_collection_shard1_replica_n1] main]
[junit4] 2> 4798534 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 4798534 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 4798534 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 4798535 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582140742752534528
[junit4] 2> 4798537 INFO (searcherExecutor-6958-thread-1-processing-n:127.0.0.1:42503_pjt x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@93a6a7[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 4798572 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 4798590 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 4798590 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:42503/pjt/control_collection_shard1_replica_n1/
[junit4] 2> 4798590 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 4798590 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:42503/pjt/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 4798590 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 4798592 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:42503/pjt/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 4798615 INFO (zkCallback-5067-thread-2-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4798643 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 4798644 INFO (qtp1614536027-20974) [n:127.0.0.1:42503_pjt c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2527
[junit4] 2> 4798728 INFO (qtp1614536027-20976) [n:127.0.0.1:42503_pjt ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 4798818 INFO (zkCallback-5067-thread-2-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 4799733 INFO (qtp1614536027-20976) [n:127.0.0.1:42503_pjt ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:42503_pjt&wt=javabin&version=2} status=0 QTime=3787
[junit4] 2> 4799782 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4799783 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46280/solr ready
[junit4] 2> 4799783 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 4799783 INFO (SocketProxy-Acceptor-42503) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=42722,localport=42503], receiveBufferSize:531000
[junit4] 2> 4799806 INFO (SocketProxy-Acceptor-42503) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=44291,localport=39908], receiveBufferSize=530904
[junit4] 2> 4799806 INFO (qtp1614536027-20973) [n:127.0.0.1:42503_pjt ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4799807 INFO (OverseerThreadFactory-6955-thread-2-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 4799807 INFO (OverseerCollectionConfigSetProcessor-98883796015644676-127.0.0.1:42503_pjt-n_0000000000) [n:127.0.0.1:42503_pjt ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 4799808 WARN (OverseerThreadFactory-6955-thread-2-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 4800054 INFO (qtp1614536027-20973) [n:127.0.0.1:42503_pjt ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 4800078 INFO (qtp1614536027-20973) [n:127.0.0.1:42503_pjt ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=272
[junit4] 2> 4800565 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-1-001 of type TLOG
[junit4] 2> 4800565 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 4800596 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6df48259{/pjt,null,AVAILABLE}
[junit4] 2> 4800597 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@53c235ce{HTTP/1.1,[http/1.1]}{127.0.0.1:35478}
[junit4] 2> 4800597 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server Started @4813280ms
[junit4] 2> 4800597 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/pjt, hostPort=45754, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-1-001/cores}
[junit4] 2> 4800597 ERROR (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4800597 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 4800597 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4800597 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 4800597 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-24T12:09:53.365Z
[junit4] 2> 4800599 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 4800599 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-1-001/solr.xml
[junit4] 2> 4800602 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 4800602 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 4800618 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 4800661 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46280/solr
[junit4] 2> 4800772 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4800787 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:45754_pjt ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 4800788 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:45754_pjt ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45754_pjt
[junit4] 2> 4800789 INFO (zkCallback-5075-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 4800789 INFO (zkCallback-5067-thread-2-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 4800825 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 4801202 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:45754_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4801240 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:45754_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4801240 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:45754_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4801242 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:45754_pjt ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-1-001/cores
[junit4] 2> 4801409 INFO (qtp1614536027-20979) [n:127.0.0.1:42503_pjt ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:45754_pjt&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4801445 INFO (OverseerCollectionConfigSetProcessor-98883796015644676-127.0.0.1:42503_pjt-n_0000000000) [n:127.0.0.1:42503_pjt ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 4801466 INFO (OverseerThreadFactory-6955-thread-3-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:45754_pjt for creating new replica
[junit4] 2> 4801478 INFO (SocketProxy-Acceptor-45754) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=43216,localport=45754], receiveBufferSize:531000
[junit4] 2> 4801490 INFO (SocketProxy-Acceptor-45754) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=35478,localport=48322], receiveBufferSize=530904
[junit4] 2> 4801495 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 4801495 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 4801661 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4802561 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 4802614 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema [collection1_shard2_replica_t41] Schema name=test
[junit4] 2> 4803363 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 4803440 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t41' using configuration from collection collection1, trusted=true
[junit4] 2> 4803441 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t41' (registry 'solr.core.collection1.shard2.replica_t41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4803441 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 4803441 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [[collection1_shard2_replica_t41] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-1-001/cores/collection1_shard2_replica_t41], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-1-001/cores/collection1_shard2_replica_t41/data/]
[junit4] 2> 4803456 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=26, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 4803472 WARN (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 4803767 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 4803767 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 4803768 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 4803768 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 4803769 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=36.0888671875, floorSegmentMB=0.9384765625, forceMergeDeletesPctAllowed=2.6420731854266144, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 4803770 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.SolrIndexSearcher Opening [Searcher@4b3bf8d4[collection1_shard2_replica_t41] main]
[junit4] 2> 4803787 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 4803788 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 4803788 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 4803788 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582140748260704256
[junit4] 2> 4803791 INFO (searcherExecutor-6969-thread-1-processing-n:127.0.0.1:45754_pjt x:collection1_shard2_replica_t41 s:shard2 c:collection1) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [collection1_shard2_replica_t41] Registered new searcher Searcher@4b3bf8d4[collection1_shard2_replica_t41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 4803805 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 4803805 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 4803805 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45754/pjt/collection1_shard2_replica_t41/
[junit4] 2> 4803805 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 4803805 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy http://127.0.0.1:45754/pjt/collection1_shard2_replica_t41/ has no replicas
[junit4] 2> 4803805 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 4803805 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController collection1_shard2_replica_t41 stopping background replication from leader
[junit4] 2> 4803807 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45754/pjt/collection1_shard2_replica_t41/ shard2
[junit4] 2> 4803844 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4803901 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 4803902 INFO (qtp1160825461-21030) [n:127.0.0.1:45754_pjt c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2407
[junit4] 2> 4803954 INFO (qtp1614536027-20979) [n:127.0.0.1:42503_pjt ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:45754_pjt&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=2545
[junit4] 2> 4804020 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4804634 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-2-001 of type TLOG
[junit4] 2> 4804635 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 4804707 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@63f9eea6{/pjt,null,AVAILABLE}
[junit4] 2> 4804707 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@64b6284c{HTTP/1.1,[http/1.1]}{127.0.0.1:44448}
[junit4] 2> 4804707 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server Started @4817390ms
[junit4] 2> 4804707 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/pjt, hostPort=34304, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-2-001/cores}
[junit4] 2> 4804707 ERROR (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4804722 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 4804723 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4804723 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 4804723 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-24T12:09:57.491Z
[junit4] 2> 4804779 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 4804779 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-2-001/solr.xml
[junit4] 2> 4804783 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 4804783 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 4804921 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 4804991 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46280/solr
[junit4] 2> 4805068 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:34304_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 4805102 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:34304_pjt ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 4805103 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:34304_pjt ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34304_pjt
[junit4] 2> 4805104 INFO (zkCallback-5075-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 4805104 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 4805117 INFO (zkCallback-5067-thread-1-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 4805141 INFO (zkCallback-5086-thread-1-processing-n:127.0.0.1:34304_pjt) [n:127.0.0.1:34304_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 4805402 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:34304_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4805442 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:34304_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4805442 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:34304_pjt ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4805459 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:34304_pjt ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-2-001/cores
[junit4] 2> 4805491 INFO (OverseerCollectionConfigSetProcessor-98883796015644676-127.0.0.1:42503_pjt-n_0000000000) [n:127.0.0.1:42503_pjt ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 4805711 INFO (SocketProxy-Acceptor-34304) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=50844,localport=34304], receiveBufferSize:531000
[junit4] 2> 4805727 INFO (SocketProxy-Acceptor-34304) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=44448,localport=46960], receiveBufferSize=530904
[junit4] 2> 4805748 INFO (qtp1003193669-21062) [n:127.0.0.1:34304_pjt ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:34304_pjt&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4805797 INFO (OverseerThreadFactory-6955-thread-4-processing-n:127.0.0.1:42503_pjt) [n:127.0.0.1:42503_pjt ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:34304_pjt for creating new replica
[junit4] 2> 4805833 INFO (SocketProxy-Acceptor-34304) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=50848,localport=34304], receiveBufferSize:531000
[junit4] 2> 4805834 INFO (SocketProxy-Acceptor-34304) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=44448,localport=46968], receiveBufferSize=530904
[junit4] 2> 4805838 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 4805839 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 4805953 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4805967 INFO (zkCallback-5086-thread-1-processing-n:127.0.0.1:34304_pjt) [n:127.0.0.1:34304_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4806936 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 4806994 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema [collection1_shard1_replica_t43] Schema name=test
[junit4] 2> 4807724 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 4807783 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t43' using configuration from collection collection1, trusted=true
[junit4] 2> 4807797 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t43' (registry 'solr.core.collection1.shard1.replica_t43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7
[junit4] 2> 4807797 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 4807797 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [[collection1_shard1_replica_t43] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-2-001/cores/collection1_shard1_replica_t43], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-2-001/cores/collection1_shard1_replica_t43/data/]
[junit4] 2> 4807853 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=26, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 4807871 WARN (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 4808034 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 4808034 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 4808035 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 4808035 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 4808053 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=36.0888671875, floorSegmentMB=0.9384765625, forceMergeDeletesPctAllowed=2.6420731854266144, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 4808053 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.SolrIndexSearcher Opening [Searcher@76149f2e[collection1_shard1_replica_t43] main]
[junit4] 2> 4808054 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 4808054 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 4808054 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 4808055 INFO (searcherExecutor-6980-thread-1-processing-n:127.0.0.1:34304_pjt x:collection1_shard1_replica_t43 s:shard1 c:collection1) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [collection1_shard1_replica_t43] Registered new searcher Searcher@76149f2e[collection1_shard1_replica_t43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 4808055 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582140752734978048
[junit4] 2> 4808071 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 4808071 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 4808071 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:34304/pjt/collection1_shard1_replica_t43/
[junit4] 2> 4808071 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 4808071 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy http://127.0.0.1:34304/pjt/collection1_shard1_replica_t43/ has no replicas
[junit4] 2> 4808071 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 4808072 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController collection1_shard1_replica_t43 stopping background replication from leader
[junit4] 2> 4808090 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:34304/pjt/collection1_shard1_replica_t43/ shard1
[junit4] 2> 4808092 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4808100 INFO (zkCallback-5086-thread-1-processing-n:127.0.0.1:34304_pjt) [n:127.0.0.1:34304_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4808140 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 4808174 INFO (qtp1003193669-21064) [n:127.0.0.1:34304_pjt c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2335
[junit4] 2> 4808176 INFO (qtp1003193669-21062) [n:127.0.0.1:34304_pjt ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:34304_pjt&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=2427
[junit4] 2> 4808322 INFO (zkCallback-5086-thread-1-processing-n:127.0.0.1:34304_pjt) [n:127.0.0.1:34304_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4808323 INFO (zkCallback-5080-thread-1-processing-n:127.0.0.1:45754_pjt) [n:127.0.0.1:45754_pjt ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 4808855 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-3-001 of type TLOG
[junit4] 2> 4808855 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 4808856 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3f402781{/pjt,null,AVAILABLE}
[junit4] 2> 4808869 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@72cfd6fe{HTTP/1.1,[http/1.1]}{127.0.0.1:39176}
[junit4] 2> 4808869 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.e.j.s.Server Started @4821552ms
[junit4] 2> 4808869 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/tempDir-001/jetty3, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/pjt, hostPort=40853, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-3-001/cores}
[junit4] 2> 4808869 ERROR (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4808870 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 4808870 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4808870 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 4808870 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-24T12:10:01.638Z
[junit4] 2> 4808938 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 4808938 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest_E99EB240D3F155EC-001/shard-3-001/solr.xml
[junit4] 2> 4808941 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 4808941 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 4808955 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@4647f1a7, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 4808992 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46280/solr
[junit4] 2> 4809138 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:40853_pjt ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 4809139 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:40853_pjt ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 4809157 INFO (TEST-LeaderFailoverAfterPartitionTest.test-seed#[E99EB240D3F155EC]) [n:127.0.0.1:40853_pjt ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40853_pjt
[junit4] 2> 4809160 I
[...truncated too long message...]
003) closing
[junit4] 2> 5666190 INFO (zkCallback-4578-thread-3-processing-n:127.0.0.1:46588_solr) [n:127.0.0.1:46588_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 5666197 WARN (OverseerAutoScalingTriggerThread-98883842714501134-127.0.0.1:46588_solr-n_0000000003) [n:127.0.0.1:46588_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 5666205 INFO (OverseerStateUpdate-98883842714501134-127.0.0.1:46588_solr-n_0000000003) [n:127.0.0.1:46588_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:46588_solr
[junit4] 2> 5666217 INFO (jetty-closer-4545-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@17049a9f{/solr,null,UNAVAILABLE}
[junit4] 2> 5666233 ERROR (SUITE-TestTlogReplica-seed#[E99EB240D3F155EC]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 5666234 INFO (SUITE-TestTlogReplica-seed#[E99EB240D3F155EC]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:42069 42069
[junit4] 2> 5666343 INFO (Thread-5913) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:42069 42069
[junit4] 2> 5666344 WARN (Thread-5913) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 32 /solr/configs/conf
[junit4] 2> 10 /solr/aliases.json
[junit4] 2> 9 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 10 /solr/clusterprops.json
[junit4] 2> 10 /solr/clusterstate.json
[junit4] 2> 6 /solr/collections/tlog_replica_test_recovery/state.json
[junit4] 2> 5 /solr/overseer_elect/election/98883842714501134-127.0.0.1:46588_solr-n_0000000003
[junit4] 2> 5 /solr/collections/tlog_replica_test_remove_leader/state.json
[junit4] 2> 4 /solr/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json
[junit4] 2> 4 /solr/collections/tlog_replica_test_kill_leader/state.json
[junit4] 2> 4 /solr/autoscaling.json
[junit4] 2> 4 /solr/collections/tlog_replica_test_recovery/leader_elect/shard1/election/98883842714501134-core_node4-n_0000000000
[junit4] 2> 4 /solr/collections/tlog_replica_test_basic_leader_election/state.json
[junit4] 2> 4 /solr/collections/tlog_replica_test_create_delete/state.json
[junit4] 2> 4 /solr/collections/tlog_replica_test_kill_tlog_replica/state.json
[junit4] 2> 3 /solr/collections/tlog_replica_test_create_delete/leader_elect/shard2/election/98883842714501134-core_node11-n_0000000000
[junit4] 2> 3 /solr/collections/tlog_replica_test_add_remove_tlog_replica/state.json
[junit4] 2> 3 /solr/collections/tlog_replica_test_only_leader_indexes/state.json
[junit4] 2> 3 /solr/collections/tlog_replica_test_delete_by_id/state.json
[junit4] 2> 2 /solr/collections/tlog_replica_test_create_delete/leader_elect/shard1/election/98883842714501134-core_node7-n_0000000001
[junit4] 2> 2 /solr/collections/tlog_replica_test_add_docs/state.json
[junit4] 2> 2 /solr/collections/tlog_replica_test_create_delete/leader_elect/shard2/election/98883842714501149-core_node10-n_0000000001
[junit4] 2> 2 /solr/collections/tlog_replica_test_real_time_get/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 10 /solr/live_nodes
[junit4] 2> 10 /solr/collections
[junit4] 2> 4 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 4 /solr/overseer/collection-queue-work
[junit4] 2> 4 /solr/overseer/queue-work
[junit4] 2> 3 /solr/overseer/queue
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestTlogReplica_E99EB240D3F155EC-001
[junit4] 2> Oct 24, 2017 12:24:33 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {foo=Lucene50(blocksize=128), title_s=PostingsFormat(name=Memory), id=PostingsFormat(name=LuceneFixedGap), foo_s=Lucene50(blocksize=128)}, docValues:{_version_=DocValuesFormat(name=Lucene70), id=DocValuesFormat(name=Memory), inplace_updatable_int=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=994, maxMBSortInHeap=7.221330154553551, sim=RandomSimilarity(queryNorm=false): {}, locale=el-CY, timezone=Africa/Bissau
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=178639080,total=518520832
[junit4] 2> NOTE: All tests run in this JVM: [TimeZoneUtilsTest, TestFieldCacheSortRandom, SuggestComponentContextFilterQueryTest, XsltUpdateRequestHandlerTest, TestGraphMLResponseWriter, HdfsBasicDistributedZkTest, BadComponentTest, CustomCollectionTest, DefaultValueUpdateProcessorTest, CoreSorterTest, TestFuzzyAnalyzedSuggestions, ZkStateReaderTest, TestRequestStatusCollectionAPI, SortByFunctionTest, TestFunctionQuery, SubstringBytesRefFilterTest, TestCoreContainer, TestClassicSimilarityFactory, TestNumericTokenStream, TestGraphTermsQParserPlugin, TestPolicyCloud, TestShortCircuitedRequests, RemoteQueryErrorTest, TestDynamicLoading, TestMultiWordSynonyms, TestImpersonationWithHadoopAuth, TestUnifiedSolrHighlighter, DistanceUnitsTest, LoggingHandlerTest, TestPullReplicaErrorHandling, RequestLoggingTest, TestPerFieldSimilarityWithDefaultOverride, TestExpandComponent, HdfsAutoAddReplicasIntegrationTest, TestLRUCache, TestConfigReload, TestMinMaxOnMultiValuedField, TestLMJelinekMercerSimilarityFactory, DistributedFacetPivotWhiteBoxTest, PKIAuthenticationIntegrationTest, TestLegacyFieldReuse, SpellCheckCollatorWithCollapseTest, TestLegacyFieldCache, HttpSolrCallGetCoreTest, SpellPossibilityIteratorTest, RequiredFieldsTest, TestCopyFieldCollectionResource, TestCloudPseudoReturnFields, TestDownShardTolerantSearch, AutoscalingHistoryHandlerTest, ConfigSetsAPITest, TestGroupingSearch, TestCoreDiscovery, TestRuleBasedAuthorizationPlugin, TestXmlQParserPlugin, SolrCoreCheckLockOnStartupTest, MissingSegmentRecoveryTest, TestSolrDeletionPolicy2, SolrMetricReporterTest, TestSolrXml, ClusterStateTest, DocValuesMissingTest, RecoveryZkTest, WrapperMergePolicyFactoryTest, SolrGraphiteReporterTest, MoveReplicaTest, TestConfigSets, CdcrUpdateLogTest, ChaosMonkeyNothingIsSafeTest, SaslZkACLProviderTest, TestRecoveryHdfs, RankQueryTest, TestSearchPerf, AutoScalingHandlerTest, DistributedExpandComponentTest, TestFiltering, TestRawResponseWriter, SuggesterWFSTTest, OverseerTest, TestMultiValuedNumericRangeQuery, TestFieldCache, FieldMutatingUpdateProcessorTest, UnloadDistributedZkTest, ScriptEngineTest, TestCollapseQParserPlugin, SparseHLLTest, TestSchemaManager, DistribJoinFromCollectionTest, TestSolrCoreParser, CollectionsAPIAsyncDistributedZkTest, TestDistribIDF, QueryElevationComponentTest, TestLRUStatsCache, SolrIndexMetricsTest, TestElisionMultitermQuery, JSONWriterTest, TestZkChroot, CoreAdminHandlerTest, CollectionReloadTest, TestLMDirichletSimilarityFactory, TestCollationFieldDocValues, TestSolrQueryResponse, TestQueryTypes, TestNamedUpdateProcessors, DocValuesNotIndexedTest, TestStreamBody, DistributedSpellCheckComponentTest, ConnectionReuseTest, TestStressCloudBlindAtomicUpdates, HdfsCollectionsAPIDistributedZkTest, SystemInfoHandlerTest, TestRebalanceLeaders, FullSolrCloudDistribCmdsTest, SuggesterTest, AtomicUpdateProcessorFactoryTest, TestBulkSchemaConcurrent, TestLeaderElectionWithEmptyReplica, ZkNodePropsTest, TestTolerantSearch, TestFieldCollectionResource, DistributedFacetPivotLongTailTest, DistanceFunctionTest, BasicDistributedZkTest, DirectUpdateHandlerTest, TestIndexSearcher, SimplePostToolTest, AnalysisAfterCoreReloadTest, TestExtendedDismaxParser, DocValuesTest, TestSurroundQueryParser, TestQueryUtils, FileBasedSpellCheckerTest, PathHierarchyTokenizerFactoryTest, MoreLikeThisHandlerTest, IndexSchemaTest, TestCSVResponseWriter, QueryParsingTest, UniqFieldsUpdateProcessorFactoryTest, CSVRequestHandlerTest, TestQuerySenderListener, AlternateDirectoryTest, CopyFieldTest, TestMergePolicyConfig, MultiTermTest, TestDocSet, TestConfig, ExternalFileFieldSortTest, TestPostingsSolrHighlighter, TestCharFilters, EchoParamsTest, TestIBSimilarityFactory, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, PluginInfoTest, PrimUtilsTest, SpellingQueryConverterTest, TestSolrJ, TestDocumentBuilder, FileUtilsTest, DistributedMLTComponentTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestDistributedMissingSort, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerConstructors, TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, AliasIntegrationTest, AsyncCallRequestStatusResponseTest, CdcrReplicationDistributedZkTest, CdcrReplicationHandlerTest, CdcrVersionReplicationTest, CleanupOldIndexTest, ConcurrentDeleteAndCreateCollectionTest, CreateCollectionCleanupTest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DeleteStatusTest, DistribCursorPagingTest, LeaderInitiatedRecoveryOnShardRestartTest, MoveReplicaHDFSFailoverTest, OverseerStatusTest, PeerSyncReplicationTest, SolrCLIZkUtilsTest, SolrXmlInZkTest, TestClusterProperties, TestCollectionAPI, TestCollectionsAPIViaSolrCloudCluster, TestConfigSetsAPIZkFailure, TestCryptoKeys, TestExclusionRuleCollectionAccess, TestLeaderInitiatedRecoveryThread, TestOnReconnectListenerSupport, TestPullReplica, TestRandomFlRTGCloud, TestRequestForwarding, TestSegmentSorting, TestSolrCloudWithDelegationTokens, TestTlogReplica]
[junit4] Completed [565/744 (3!)] on J2 in 176.56s, 13 tests, 1 error <<< FAILURES!
[...truncated 47830 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 191 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/191/
6 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.BasicDistributedZkTest
Error Message:
118 threads leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest: 1) Thread[id=15078, name=qtp1220744071-15078, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=15125, name=watches-2817-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=15157, name=Scheduler-1212435602, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=15177, name=zkCallback-2823-thread-2, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=15186, name=zkCallback-2810-thread-6, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=15179, name=zkCallback-2823-thread-4, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=15213, name=coreZkRegister-4658-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=15131, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=15203, name=searcherExecutor-4697-thread-1, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=15076, name=qtp1220744071-15076, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=15158, name=watches-2824-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=15174, name=zkCallback-2816-thread-3, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=15175, name=zkCallback-2816-thread-4, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=15075, name=qtp1220744071-15075, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=15105, name=qtp367347145-15105, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=15169, name=zkCallback-2804-thread-5, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=15215, name=recoveryExecutor-2802-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=15042, name=qtp990776779-15042, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Object.wait(Native Method) at org.apache.solr.cloud.OverseerTaskQueue$LatchWatcher.await(OverseerTaskQueue.java:149) at org.apache.solr.cloud.OverseerTaskQueue.offer(OverseerTaskQueue.java:199) at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:300) at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:244) at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:222) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) at org.eclipse.jetty.server.Server.handle(Server.java:534) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=15194, name=zkCallback-2810-thread-1-processing-n:127.0.0.1:53790_ufox%2Fj-SendThread(127.0.0.1:39256), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 20) Thread[id=15172, name=zkCallback-2810-thread-4, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=15192, name=zkCallback-2816-thread-3-processing-n:127.0.0.1:40195_ufox%2Fj-SendThread(127.0.0.1:39256), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 22) Thread[id=15114, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=15108, name=qtp367347145-15108, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=15049, name=qtp990776779-15049, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=15195, name=zkCallback-2810-thread-1-processing-n:127.0.0.1:53790_ufox%2Fj-EventThread, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 26) Thread[id=15113, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=15137, name=qtp183297797-15137-acceptor-0@51de79c-ServerConnector@63305544{HTTP/1.1,[http/1.1]}{127.0.0.1:53551}, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=15050, name=org.eclipse.jetty.server.session.HashSessionManager@ad95060Timer, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=15101, name=qtp367347145-15101, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=15102, name=qtp367347145-15102, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=15103, name=qtp367347145-15103-acceptor-0@349ab6a8-ServerConnector@359b5045{HTTP/1.1,[http/1.1]}{127.0.0.1:40195}, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=15234, name=zkCallback-2823-thread-7, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=15204, name=coreZkRegister-4669-thread-2, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=15182, name=zkCallback-2816-thread-6, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=15046, name=qtp990776779-15046, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=15227, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 37) Thread[id=15045, name=qtp990776779-15045, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 38) Thread[id=15154, name=zkCallback-2823-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=15233, name=zkCallback-2804-thread-8, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 40) Thread[id=15055, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 41) Thread[id=15229, name=ScheduledTrigger-4705-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:389) at org.apache.solr.cloud.ZkDistributedQueue.offer(ZkDistributedQueue.java:293) at org.apache.solr.cloud.autoscaling.TriggerEventQueue.offerEvent(TriggerEventQueue.java:58) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.enqueue(ScheduledTriggers.java:411) at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:205) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$145/1884738894.process(Unknown Source) at org.apache.solr.cloud.autoscaling.NodeLostTrigger.run(NodeLostTrigger.java:163) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:453) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 42) Thread[id=15173, name=zkCallback-2816-thread-2, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 43) Thread[id=15109, name=org.eclipse.jetty.server.session.HashSessionManager@392a8cc4Timer, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 44) Thread[id=15120, name=zkCallback-2816-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 45) Thread[id=15142, name=qtp183297797-15142, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 46) Thread[id=15207, name=updateExecutor-2820-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 47) Thread[id=15107, name=qtp367347145-15107, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 48) Thread[id=15232, name=zkCallback-2816-thread-7, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 49) Thread[id=15148, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 50) Thread[id=15121, name=Thread-4550, state=WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 51) Thread[id=15143, name=org.eclipse.jetty.server.session.HashSessionManager@35deb332Timer, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 52) Thread[id=15092, name=Thread-4541, state=WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 53) Thread[id=15104, name=qtp367347145-15104, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 54) Thread[id=15071, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 55) Thread[id=15048, name=qtp990776779-15048, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 56) Thread[id=15141, name=qtp183297797-15141, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 57) Thread[id=15084, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 58) Thread[id=15054, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 59) Thread[id=15228, name=OverseerAutoScalingTriggerThread-98877173344305170-127.0.0.1:53551_ufox%2Fj-n_0000000004, state=WAITING, group=Overseer autoscaling triggers] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:161) at java.lang.Thread.run(Thread.java:748) 60) Thread[id=15138, name=qtp183297797-15138, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 61) Thread[id=15237, name=ScheduledTrigger-4705-thread-4, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 62) Thread[id=15085, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 63) Thread[id=15043, name=qtp990776779-15043, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 64) Thread[id=15196, name=zkCallback-2804-thread-3-processing-n:127.0.0.1:35580_ufox%2Fj-SendThread(127.0.0.1:39256), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 65) Thread[id=15166, name=zkCallback-2804-thread-2, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 66) Thread[id=15098, name=searcherExecutor-4675-thread-1, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 67) Thread[id=15135, name=qtp183297797-15135, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 68) Thread[id=15062, name=Thread-4532, state=WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 69) Thread[id=15167, name=zkCallback-2804-thread-3, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 70) Thread[id=15170, name=zkCallback-2810-thread-2, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 71) Thread[id=15199, name=coreZkRegister-4680-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 72) Thread[id=15208, name=recoveryExecutor-2821-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 73) Thread[id=15231, name=ScheduledTrigger-4705-thread-2, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 74) Thread[id=15047, name=qtp990776779-15047-acceptor-0@5af199d1-ServerConnector@2bf37a8d{HTTP/1.1,[http/1.1]}{127.0.0.1:35580}, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 75) Thread[id=15091, name=zkCallback-2810-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 76) Thread[id=15077, name=qtp1220744071-15077, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 77) Thread[id=15100, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 78) Thread[id=15067, name=watches-2805-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 79) Thread[id=15183, name=zkCallback-2804-thread-6, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 80) Thread[id=15080, name=org.eclipse.jetty.server.session.HashSessionManager@2dc7e136Timer, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 81) Thread[id=15168, name=zkCallback-2804-thread-4, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 82) Thread[id=15197, name=zkCallback-2804-thread-3-processing-n:127.0.0.1:35580_ufox%2Fj-EventThread, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 83) Thread[id=15171, name=zkCallback-2810-thread-3, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 84) Thread[id=15096, name=watches-2811-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 85) Thread[id=15226, name=OverseerCollectionConfigSetProcessor-98877173344305170-127.0.0.1:53551_ufox%2Fj-n_0000000004, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 86) Thread[id=15136, name=qtp183297797-15136, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 87) Thread[id=15216, name=coreZkRegister-4658-thread-2, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 88) Thread[id=15139, name=qtp183297797-15139, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 89) Thread[id=15069, name=searcherExecutor-4664-thread-1, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 90) Thread[id=15061, name=zkCallback-2804-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 91) Thread[id=15184, name=zkCallback-2804-thread-7, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 92) Thread[id=15225, name=OverseerStateUpdate-98877173344305170-127.0.0.1:53551_ufox%2Fj-n_0000000004, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:202) at java.lang.Thread.run(Thread.java:748) 93) Thread[id=15180, name=zkCallback-2823-thread-5, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 94) Thread[id=15181, name=zkCallback-2823-thread-6, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 95) Thread[id=15193, name=zkCallback-2816-thread-3-processing-n:127.0.0.1:40195_ufox%2Fj-EventThread, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 96) Thread[id=15124, name=Scheduler-1024594496, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 97) Thread[id=15079, name=qtp1220744071-15079, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 98) Thread[id=15236, name=ScheduledTrigger-4705-thread-3, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 99) Thread[id=15106, name=qtp367347145-15106, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 100) Thread[id=15072, name=qtp1220744071-15072, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 101) Thread[id=15095, name=Scheduler-1597017116, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 102) Thread[id=15155, name=Thread-4559, state=WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 103) Thread[id=15185, name=zkCallback-2810-thread-5, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 104) Thread[id=15176, name=zkCallback-2816-thread-5, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 105) Thread[id=15073, name=qtp1220744071-15073, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 106) Thread[id=15152, name=TEST-BasicDistributedZkTest.test-seed#[E0ED00A999BBA103]-SendThread(127.0.0.1:39256), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 107) Thread[id=15074, name=qtp1220744071-15074-acceptor-0@ea51341-ServerConnector@35fb7f9f{HTTP/1.1,[http/1.1]}{127.0.0.1:53790}, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 108) Thread[id=15178, name=zkCallback-2823-thread-3, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 109) Thread[id=15044, name=qtp990776779-15044, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 110) Thread[id=15127, name=searcherExecutor-4686-thread-1, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 111) Thread[id=15140, name=qtp183297797-15140, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 112) Thread[id=15064, name=Scheduler-238212817, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 113) Thread[id=15153, name=TEST-BasicDistributedZkTest.test-seed#[E0ED00A999BBA103]-EventThread, state=WAITING, group=TGRP-BasicDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[...truncated too long message...]
, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 66) Thread[id=15178, name=zkCallback-2823-thread-3, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 67) Thread[id=15127, name=searcherExecutor-4686-thread-1, state=WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 68) Thread[id=15140, name=qtp183297797-15140, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 69) Thread[id=15092, name=Thread-4541, state=WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678)
[junit4] > 70) Thread[id=15064, name=Scheduler-238212817, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 71) Thread[id=15200, name=coreZkRegister-4680-thread-2, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 72) Thread[id=15214, name=updateExecutor-2801-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 73) Thread[id=15202, name=coreZkRegister-4669-thread-1, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 74) Thread[id=15235, name=zkCallback-2823-thread-8, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([E0ED00A999BBA103]:0)
[junit4] Completed [352/744 (3!)] on J0 in 127.38s, 1 test, 3 errors <<< FAILURES!
[...truncated 48429 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 190 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/190/
4 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
Error Message:
8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest: 1) Thread[id=644, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1038, name=zkCallback-113-thread-1-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 3) Thread[id=1037, name=zkCallback-113-thread-1-SendThread(127.0.0.1:57157), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest:
1) Thread[id=644, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=1038, name=zkCallback-113-thread-1-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
3) Thread[id=1037, name=zkCallback-113-thread-1-SendThread(127.0.0.1:57157), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
at __randomizedtesting.SeedInfo.seed([96FB05935602FDD4]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=1037, name=zkCallback-113-thread-1-SendThread(127.0.0.1:57157), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=1037, name=zkCallback-113-thread-1-SendThread(127.0.0.1:57157), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
at __randomizedtesting.SeedInfo.seed([96FB05935602FDD4]:0)
FAILED: org.apache.solr.client.solrj.io.stream.SelectWithEvaluatorsTest.testSelectWithEvaluatorsStream
Error Message:
Collection not found: collection1
Stack Trace:
org.apache.solr.common.SolrException: Collection not found: collection1
at __randomizedtesting.SeedInfo.seed([E076EBD28405821B:6C2E0AA9D97CD2B8]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:834)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
at org.apache.solr.client.solrj.io.stream.SelectWithEvaluatorsTest.cleanIndex(SelectWithEvaluatorsTest.java:83)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test
Error Message:
KeeperErrorCode = Session expired for /live_nodes
Stack Trace:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /live_nodes
at __randomizedtesting.SeedInfo.seed([96FB05935602FDD4:1EAF3A49F8FE902C]:0)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1532)
at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:340)
at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:337)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:337)
at org.apache.solr.common.cloud.ZkStateReader.refreshLiveNodes(ZkStateReader.java:718)
at org.apache.solr.common.cloud.ZkStateReader.updateLiveNodes(ZkStateReader.java:386)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.logReplicaTypesReplicationInfo(AbstractFullDistribZkTestBase.java:2067)
at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:312)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11819 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
[junit4] 2> 342156 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[96FB05935602FDD4]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/init-core-data-001
[junit4] 2> 342209 WARN (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[96FB05935602FDD4]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=25 numCloses=25
[junit4] 2> 342229 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[96FB05935602FDD4]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 342231 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[96FB05935602FDD4]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 342231 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[96FB05935602FDD4]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /eov/db
[junit4] 2> 342250 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ChaosMonkeyNothingIsSafeWithPullReplicasTest Starting ChaosMonkey test with 2 shards and 6 nodes
[junit4] 2> 342251 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 342473 INFO (Thread-129) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 342473 INFO (Thread-129) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 342481 ERROR (Thread-129) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 342577 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ZkTestServer start zk server on port:57157
[junit4] 2> 342794 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 342809 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 342810 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 342811 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 342812 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 342829 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 342832 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 342833 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 342854 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 342856 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 342873 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 342921 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 345578 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 345657 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@31cdc7c7{/eov/db,null,AVAILABLE}
[junit4] 2> 345659 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5ea941cf{HTTP/1.1,[http/1.1]}{127.0.0.1:58799}
[junit4] 2> 345659 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server Started @366528ms
[junit4] 2> 345659 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/tempDir-001/control/data, hostContext=/eov/db, hostPort=58799, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/control-001/cores}
[junit4] 2> 345659 ERROR (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 345659 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 345659 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 345659 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 345659 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-22T00:54:58.604Z
[junit4] 2> 346262 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 346262 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/control-001/solr.xml
[junit4] 2> 346444 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 346444 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 346461 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 346466 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57157/solr
[junit4] 2> 346919 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 346920 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:58799_eov%2Fdb
[junit4] 2> 346921 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.Overseer Overseer (id=98869817863176196-127.0.0.1:58799_eov%2Fdb-n_0000000000) starting
[junit4] 2> 348006 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58799_eov%2Fdb
[junit4] 2> 348069 INFO (OverseerStateUpdate-98869817863176196-127.0.0.1:58799_eov%2Fdb-n_0000000000) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 348893 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 348934 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 348935 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 348936 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/control-001/cores
[junit4] 2> 349520 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 349537 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57157/solr ready
[junit4] 2> 349558 INFO (qtp1833550070-605) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:58799_eov%252Fdb&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 349635 INFO (OverseerThreadFactory-230-thread-1-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 349974 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 349974 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 350117 INFO (zkCallback-105-thread-1-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 351129 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 351238 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 353516 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 353688 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 353689 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 353762 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 353762 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 353943 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=30.818359375, floorSegmentMB=1.5107421875, forceMergeDeletesPctAllowed=22.137981077966007, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8867546865563619
[junit4] 2> 354037 WARN (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 355165 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 355165 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 355166 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 355166 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 355168 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=87148644, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 355243 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4029d23f[control_collection_shard1_replica_n1] main]
[junit4] 2> 355293 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 355294 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 355295 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 355449 INFO (searcherExecutor-233-thread-1-processing-n:127.0.0.1:58799_eov%2Fdb x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@4029d23f[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 355449 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581917099412946944
[junit4] 2> 355505 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 355505 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 355505 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:58799/eov/db/control_collection_shard1_replica_n1/
[junit4] 2> 355505 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 355505 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:58799/eov/db/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 355505 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 355520 INFO (zkCallback-105-thread-2-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 355521 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:58799/eov/db/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 355521 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 355578 INFO (qtp1833550070-603) [n:127.0.0.1:58799_eov%2Fdb c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=5604
[junit4] 2> 355637 INFO (qtp1833550070-605) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 355681 INFO (zkCallback-105-thread-2-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 356647 INFO (qtp1833550070-605) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:58799_eov%252Fdb&wt=javabin&version=2} status=0 QTime=7089
[junit4] 2> 357015 INFO (OverseerCollectionConfigSetProcessor-98869817863176196-127.0.0.1:58799_eov%2Fdb-n_0000000000) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 357020 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 357021 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57157/solr ready
[junit4] 2> 357021 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 357026 INFO (qtp1833550070-606) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 357094 INFO (OverseerThreadFactory-230-thread-2-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 357113 WARN (OverseerThreadFactory-230-thread-2-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 357253 INFO (qtp1833550070-606) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 357253 INFO (qtp1833550070-606) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=227
[junit4] 2> 357276 INFO (OverseerCollectionConfigSetProcessor-98869817863176196-127.0.0.1:58799_eov%2Fdb-n_0000000000) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 359123 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-1-001 of type NRT
[junit4] 2> 359138 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 359223 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@ed91ae4{/eov/db,null,AVAILABLE}
[junit4] 2> 359223 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2c089a0f{HTTP/1.1,[http/1.1]}{127.0.0.1:47004}
[junit4] 2> 359223 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server Started @380092ms
[junit4] 2> 359223 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/eov/db, hostPort=47004, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-1-001/cores}
[junit4] 2> 359223 ERROR (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 359224 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 359224 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 359224 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 359224 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-22T00:55:12.169Z
[junit4] 2> 359269 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 359269 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-1-001/solr.xml
[junit4] 2> 359272 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 359272 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 359494 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 359505 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57157/solr
[junit4] 2> 359718 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 359721 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 359733 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47004_eov%2Fdb
[junit4] 2> 359734 INFO (zkCallback-105-thread-2-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 359805 INFO (zkCallback-113-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 361115 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 361241 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 361241 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 361243 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-1-001/cores
[junit4] 2> 361626 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 361986 INFO (qtp1297987795-652) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:47004_eov%252Fdb&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 362016 INFO (OverseerThreadFactory-230-thread-3-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:47004_eov%2Fdb for creating new replica
[junit4] 2> 362183 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 362184 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 362275 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 363375 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 363618 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema [collection1_shard2_replica_n41] Schema name=test
[junit4] 2> 364606 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 366350 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n41' using configuration from collection collection1, trusted=true
[junit4] 2> 366351 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n41' (registry 'solr.core.collection1.shard2.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 366351 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 366351 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [[collection1_shard2_replica_n41] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-1-001/cores/collection1_shard2_replica_n41], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-1-001/cores/collection1_shard2_replica_n41/data/]
[junit4] 2> 366394 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=30.818359375, floorSegmentMB=1.5107421875, forceMergeDeletesPctAllowed=22.137981077966007, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8867546865563619
[junit4] 2> 367348 WARN (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 367815 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 367815 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 367816 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 367816 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 367849 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=87148644, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 368050 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.SolrIndexSearcher Opening [Searcher@1ecc0a46[collection1_shard2_replica_n41] main]
[junit4] 2> 368052 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 368052 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 368053 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 368055 INFO (searcherExecutor-244-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb x:collection1_shard2_replica_n41 s:shard2 c:collection1) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [collection1_shard2_replica_n41] Registered new searcher Searcher@1ecc0a46[collection1_shard2_replica_n41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 368055 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581917112631296000
[junit4] 2> 368060 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 368060 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 368060 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:47004/eov/db/collection1_shard2_replica_n41/
[junit4] 2> 368060 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 368060 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy http://127.0.0.1:47004/eov/db/collection1_shard2_replica_n41/ has no replicas
[junit4] 2> 368060 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 368064 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:47004/eov/db/collection1_shard2_replica_n41/ shard2
[junit4] 2> 368113 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 368114 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 368115 INFO (qtp1297987795-650) [n:127.0.0.1:47004_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=5932
[junit4] 2> 368119 INFO (qtp1297987795-652) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:47004_eov%252Fdb&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=6132
[junit4] 2> 368217 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 368353 INFO (OverseerCollectionConfigSetProcessor-98869817863176196-127.0.0.1:58799_eov%2Fdb-n_0000000000) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 370037 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-2-001 of type NRT
[junit4] 2> 370042 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 370046 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4b799471{/eov/db,null,AVAILABLE}
[junit4] 2> 370046 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@18c6e2e4{HTTP/1.1,[http/1.1]}{127.0.0.1:50167}
[junit4] 2> 370046 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server Started @390915ms
[junit4] 2> 370046 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/eov/db, hostPort=50167, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-2-001/cores}
[junit4] 2> 370046 ERROR (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 370085 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 370085 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 370085 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 370085 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-22T00:55:23.030Z
[junit4] 2> 370142 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 370142 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-2-001/solr.xml
[junit4] 2> 370146 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 370146 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 370147 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 370174 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57157/solr
[junit4] 2> 370285 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 370290 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 370308 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50167_eov%2Fdb
[junit4] 2> 370308 INFO (zkCallback-105-thread-2-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 370309 INFO (zkCallback-113-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 370325 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 370445 INFO (zkCallback-124-thread-2-processing-n:127.0.0.1:50167_eov%2Fdb) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 371131 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 371230 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 371230 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 371232 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-2-001/cores
[junit4] 2> 372110 INFO (qtp1297987795-653) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:50167_eov%252Fdb&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 372144 INFO (OverseerThreadFactory-230-thread-4-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:50167_eov%2Fdb for creating new replica
[junit4] 2> 372270 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 372270 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 372289 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 372375 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 372575 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.IndexSchema [collection1_shard1_replica_n43] Schema name=test
[junit4] 2> 373764 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 374166 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n43' using configuration from collection collection1, trusted=true
[junit4] 2> 374227 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n43' (registry 'solr.core.collection1.shard1.replica_n43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 374227 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 374228 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrCore [[collection1_shard1_replica_n43] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-2-001/cores/collection1_shard1_replica_n43], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-2-001/cores/collection1_shard1_replica_n43/data/]
[junit4] 2> 374255 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=30.818359375, floorSegmentMB=1.5107421875, forceMergeDeletesPctAllowed=22.137981077966007, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8867546865563619
[junit4] 2> 374358 WARN (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 376057 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 376057 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 376058 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 376058 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 376060 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=87148644, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 376145 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.SolrIndexSearcher Opening [Searcher@5182bff6[collection1_shard1_replica_n43] main]
[junit4] 2> 376157 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 376158 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 376158 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 376170 INFO (searcherExecutor-255-thread-1-processing-n:127.0.0.1:50167_eov%2Fdb x:collection1_shard1_replica_n43 s:shard1 c:collection1) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrCore [collection1_shard1_replica_n43] Registered new searcher Searcher@5182bff6[collection1_shard1_replica_n43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 376173 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581917121143635968
[junit4] 2> 376195 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 376195 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 376195 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:50167/eov/db/collection1_shard1_replica_n43/
[junit4] 2> 376195 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 376195 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy http://127.0.0.1:50167/eov/db/collection1_shard1_replica_n43/ has no replicas
[junit4] 2> 376195 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 376217 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 376234 INFO (zkCallback-124-thread-2-processing-n:127.0.0.1:50167_eov%2Fdb) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 376235 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:50167/eov/db/collection1_shard1_replica_n43/ shard1
[junit4] 2> 376235 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 376253 INFO (qtp296038939-683) [n:127.0.0.1:50167_eov%2Fdb c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3983
[junit4] 2> 376270 INFO (qtp1297987795-653) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:50167_eov%252Fdb&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=4160
[junit4] 2> 376381 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 376382 INFO (zkCallback-124-thread-2-processing-n:127.0.0.1:50167_eov%2Fdb) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 377075 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-3-001 of type NRT
[junit4] 2> 377112 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 377157 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@290ab17c{/eov/db,null,AVAILABLE}
[junit4] 2> 377158 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@ceeb05c{HTTP/1.1,[http/1.1]}{127.0.0.1:42402}
[junit4] 2> 377158 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.e.j.s.Server Started @398027ms
[junit4] 2> 377158 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/eov/db, hostPort=42402, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-3-001/cores}
[junit4] 2> 377158 ERROR (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 377250 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 377250 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 377250 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 377250 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-22T00:55:30.195Z
[junit4] 2> 377327 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 377327 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-3-001/solr.xml
[junit4] 2> 377331 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 377331 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 377332 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 377434 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57157/solr
[junit4] 2> 377565 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 377567 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 377568 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42402_eov%2Fdb
[junit4] 2> 377577 INFO (zkCallback-124-thread-2-processing-n:127.0.0.1:50167_eov%2Fdb) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 377593 INFO (zkCallback-113-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 377578 INFO (zkCallback-105-thread-1-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 377577 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 377757 INFO (zkCallback-130-thread-2-processing-n:127.0.0.1:42402_eov%2Fdb) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 378141 INFO (OverseerCollectionConfigSetProcessor-98869817863176196-127.0.0.1:58799_eov%2Fdb-n_0000000000) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 379334 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 380496 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 380496 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 380554 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[96FB05935602FDD4]) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-3-001/cores
[junit4] 2> 381138 INFO (qtp1841800575-713) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:42402_eov%252Fdb&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 381199 INFO (OverseerThreadFactory-230-thread-5-processing-n:127.0.0.1:58799_eov%2Fdb) [n:127.0.0.1:58799_eov%2Fdb ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:42402_eov%2Fdb for creating new replica
[junit4] 2> 381246 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 381253 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 381286 INFO (zkCallback-124-thread-2-processing-n:127.0.0.1:50167_eov%2Fdb) [n:127.0.0.1:50167_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 381286 INFO (zkCallback-118-thread-1-processing-n:127.0.0.1:47004_eov%2Fdb) [n:127.0.0.1:47004_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 381301 INFO (zkCallback-130-thread-2-processing-n:127.0.0.1:42402_eov%2Fdb) [n:127.0.0.1:42402_eov%2Fdb ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 382769 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 382911 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.IndexSchema [collection1_shard2_replica_n45] Schema name=test
[junit4] 2> 384076 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 384181 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n45' using configuration from collection collection1, trusted=true
[junit4] 2> 384257 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n45' (registry 'solr.core.collection1.shard2.replica_n45') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6836986d
[junit4] 2> 384257 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 384257 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrCore [[collection1_shard2_replica_n45] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-3-001/cores/collection1_shard2_replica_n45], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_96FB05935602FDD4-001/shard-3-001/cores/collection1_shard2_replica_n45/data/]
[junit4] 2> 384282 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=30.818359375, floorSegmentMB=1.5107421875, forceMergeDeletesPctAllowed=22.137981077966007, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8867546865563619
[junit4] 2> 384373 WARN (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 385117 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 385117 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 385118 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 385118 INFO (qtp1841800575-711) [n:127.0.0.1:42402_eov%2Fdb c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 385120 INFO (qtp1841800575-711
[...truncated too long message...]
kStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 622607 INFO (zkCallback-139-thread-2-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 622705 INFO (coreCloseExecutor-509-thread-1) [n:127.0.0.1:36943_solr c:collection1_collection s:shard2 r:core_node4 x:collection1_collection_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1_collection.shard2.leader, tag=576650243
[junit4] 2> 622742 INFO (zkCallback-139-thread-3-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 622742 INFO (zkCallback-145-thread-1-processing-n:127.0.0.1:49113_solr) [n:127.0.0.1:49113_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 622742 INFO (zkCallback-145-thread-3-processing-n:127.0.0.1:49113_solr) [n:127.0.0.1:49113_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 622747 INFO (zkCallback-143-thread-2-processing-n:127.0.0.1:54044_solr) [n:127.0.0.1:54044_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (0)
[junit4] 2> 622750 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f41cb1f1e000a, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 622756 INFO (zkCallback-139-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 622790 INFO (jetty-closer-124-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@10ae52e9{/solr,null,UNAVAILABLE}
[junit4] 2> 622856 INFO (coreCloseExecutor-511-thread-1) [n:127.0.0.1:49113_solr c:collection1_collection s:shard1 r:core_node3 x:collection1_collection_shard1_replica_n1] o.a.s.c.SolrCore [collection1_collection_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@27c1338e
[junit4] 2> 622856 INFO (coreCloseExecutor-511-thread-1) [n:127.0.0.1:49113_solr c:collection1_collection s:shard1 r:core_node3 x:collection1_collection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1_collection.shard1.replica_n1, tag=666973070
[junit4] 2> 622856 INFO (coreCloseExecutor-511-thread-1) [n:127.0.0.1:49113_solr c:collection1_collection s:shard1 r:core_node3 x:collection1_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3e64835: rootName = solr_49113, domain = solr.core.collection1_collection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.collection1_collection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@6900f402
[junit4] 2> 622918 INFO (zkCallback-145-thread-2-processing-n:127.0.0.1:49113_solr) [n:127.0.0.1:49113_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 622919 INFO (zkCallback-139-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 622986 INFO (zkCallback-145-thread-3-processing-n:127.0.0.1:49113_solr) [n:127.0.0.1:49113_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 623041 INFO (zkCallback-139-thread-3-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1_collection/state.json] for collection [collection1_collection] has occurred - updating... (live nodes size: [0])
[junit4] 2> 623042 WARN (zkCallback-139-thread-3-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /collections/collection1_collection/state.json]
[junit4] 2> 623056 INFO (coreCloseExecutor-511-thread-1) [n:127.0.0.1:49113_solr c:collection1_collection s:shard1 r:core_node3 x:collection1_collection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1_collection.shard1.leader, tag=666973070
[junit4] 2> 623070 INFO (jetty-closer-124-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@59eeb86c{/solr,null,UNAVAILABLE}
[junit4] 2> 623201 INFO (jetty-closer-124-thread-2) [ ] o.a.s.c.Overseer Overseer (id=98870057481666572-127.0.0.1:49113_solr-n_0000000000) closing
[junit4] 2> 623201 INFO (OverseerStateUpdate-98870057481666572-127.0.0.1:49113_solr-n_0000000000) [n:127.0.0.1:49113_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:49113_solr
[junit4] 2> 623299 WARN (OverseerAutoScalingTriggerThread-98870057481666572-127.0.0.1:49113_solr-n_0000000000) [n:127.0.0.1:49113_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 623356 INFO (jetty-closer-124-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@183525b4{/solr,null,UNAVAILABLE}
[junit4] 2> 624294 ERROR (SUITE-SelectWithEvaluatorsTest-seed#[E076EBD28405821B]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 624294 INFO (SUITE-SelectWithEvaluatorsTest-seed#[E076EBD28405821B]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39692 39692
[junit4] 2> 624911 INFO (Thread-243) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39692 39692
[junit4] 2> 624911 WARN (Thread-243) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/aliases.json
[junit4] 2> 5 /solr/clusterprops.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 2 /solr/configs/conf
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 16 /solr/collections/collection1_collection/state.json
[junit4] 2> 5 /solr/clusterstate.json
[junit4] 2> 2 /solr/overseer_elect/election/98870057481666572-127.0.0.1:49113_solr-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/live_nodes
[junit4] 2> 5 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.SelectWithEvaluatorsTest_E076EBD28405821B-001
[junit4] 2> Oct 22, 2017 1:56:17 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1672, maxMBSortInHeap=6.692899709764929, sim=RandomSimilarity(queryNorm=true): {}, locale=mk-MK, timezone=Etc/GMT-5
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=283037352,total=470286336
[junit4] 2> NOTE: All tests run in this JVM: [SolrQueryTest, TestCollectionAdminRequest, NormalizeEvaluatorTest, TestJsonRecordReader, HyperbolicSineEvaluatorTest, PowerEvaluatorTest, JdbcDriverTest, TestSolrJErrorHandling, CloudSolrClientTest, QueryResponseTest, UuidEvaluatorTest, RegressionEvaluatorTest, TestDocumentObjectBinder, GetByIdTest, CommonParamsTest, LessThanEqualToEvaluatorTest, TestSuggesterResponse, FacetFieldTest, ClientUtilsTest, SolrExampleEmbeddedTest, TestValidatingJsonMap, LBHttpSolrClientBuilderTest, TestCloudSolrClientConnections, LargeVolumeBinaryJettyTest, LargeVolumeEmbeddedTest, TestSolrProperties, MergeIndexesEmbeddedTest, NamedListTest, AnlysisResponseBaseTest, TestLBHttpSolrClient, SelectWithEvaluatorsTest]
[junit4] Completed [101/145 (1!)] on J1 in 26.95s, 1 test, 1 error <<< FAILURES!
[...truncated 46436 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 189 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/189/
6 tests failed.
FAILED: org.apache.solr.cloud.CdcrBootstrapTest.testBootstrapWithSourceCluster
Error Message:
Captured an uncaught exception in thread: Thread[id=848, name=Thread-154, state=RUNNABLE, group=TGRP-CdcrBootstrapTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=848, name=Thread-154, state=RUNNABLE, group=TGRP-CdcrBootstrapTest]
at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4:E4158E6F316E538E]:0)
Caused by: java.lang.AssertionError: 1
at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4]:0)
at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:192)
at org.apache.solr.core.SolrCore.close(SolrCore.java:1614)
at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:870)
at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1285)
at org.apache.solr.handler.IndexFetcher.lambda$reloadCore$0(IndexFetcher.java:910)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.CdcrBootstrapTest
Error Message:
ObjectTracker found 3 object(s) that were not released!!! [SolrCore, MockDirectoryWrapper, RawDirectoryWrapper] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1020) at org.apache.solr.core.SolrCore.reload(SolrCore.java:637) at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1284) at org.apache.solr.handler.IndexFetcher.lambda$reloadCore$0(IndexFetcher.java:910) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:494) at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:338) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:421) at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:768) at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:723) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:92) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:742) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:935) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:844) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1036) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) at org.eclipse.jetty.server.Server.handle(Server.java:534) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 3 object(s) that were not released!!! [SolrCore, MockDirectoryWrapper, RawDirectoryWrapper]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1020)
at org.apache.solr.core.SolrCore.reload(SolrCore.java:637)
at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1284)
at org.apache.solr.handler.IndexFetcher.lambda$reloadCore$0(IndexFetcher.java:910)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.MockDirectoryWrapper
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:494)
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:338)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:421)
at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:768)
at org.apache.solr.handler.CdcrRequestHandler$BootstrapCallable.call(CdcrRequestHandler.java:723)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.RawDirectoryWrapper
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:92)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:742)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:935)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:844)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1036)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:295)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestHdfsCloudBackupRestore.test
Error Message:
expected:<COMPLETED> but was:<RUNNING>
Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4:B517E0719CF62D3C]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:277)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.testSslAndClientAuth
Error Message:
Could not find collection:first_collection
Stack Trace:
java.lang.AssertionError: Could not find collection:first_collection
at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4:EEC73666A0C71B38]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNotNull(Assert.java:526)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.checkCreateCollection(TestMiniSolrCloudClusterSSL.java:202)
at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.checkClusterWithCollectionCreations(TestMiniSolrCloudClusterSSL.java:172)
at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.checkClusterWithNodeReplacement(TestMiniSolrCloudClusterSSL.java:138)
at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.testSslAndClientAuth(TestMiniSolrCloudClusterSSL.java:120)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestMiniSolrCloudClusterSSL
Error Message:
9 threads leaked from SUITE scope at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL: 1) Thread[id=9145, name=OverseerAutoScalingTriggerThread-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=WAITING, group=Overseer autoscaling triggers] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:161) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=9053, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=9146, name=ScheduledTrigger-4119-thread-1, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:389) at org.apache.solr.cloud.ZkDistributedQueue.offer(ZkDistributedQueue.java:293) at org.apache.solr.cloud.autoscaling.TriggerEventQueue.offerEvent(TriggerEventQueue.java:58) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.enqueue(ScheduledTriggers.java:411) at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:205) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$237/524708820.process(Unknown Source) at org.apache.solr.cloud.autoscaling.NodeLostTrigger.run(NodeLostTrigger.java:163) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:453) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=9076, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=9143, name=OverseerStateUpdate-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:202) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=9144, name=OverseerCollectionConfigSetProcessor-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=9108, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-EventThread, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 8) Thread[id=9054, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=9107, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-SendThread(127.0.0.1:48343), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL:
1) Thread[id=9145, name=OverseerAutoScalingTriggerThread-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=WAITING, group=Overseer autoscaling triggers]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:161)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=9053, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=9146, name=ScheduledTrigger-4119-thread-1, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:389)
at org.apache.solr.cloud.ZkDistributedQueue.offer(ZkDistributedQueue.java:293)
at org.apache.solr.cloud.autoscaling.TriggerEventQueue.offerEvent(TriggerEventQueue.java:58)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.enqueue(ScheduledTriggers.java:411)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:205)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$237/524708820.process(Unknown Source)
at org.apache.solr.cloud.autoscaling.NodeLostTrigger.run(NodeLostTrigger.java:163)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:453)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=9076, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=9143, name=OverseerStateUpdate-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=TIMED_WAITING, group=Overseer state updater.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:202)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=9144, name=OverseerCollectionConfigSetProcessor-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=TIMED_WAITING, group=Overseer collection creation process.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)
at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=9108, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-EventThread, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
8) Thread[id=9054, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=9107, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-SendThread(127.0.0.1:48343), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestMiniSolrCloudClusterSSL
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=9146, name=ScheduledTrigger-4119-thread-1, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=9107, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-SendThread(127.0.0.1:48343), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=9146, name=ScheduledTrigger-4119-thread-1, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=9107, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-SendThread(127.0.0.1:48343), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4]:0)
Build Log:
[...truncated 11625 lines...]
[junit4] Suite: org.apache.solr.cloud.CdcrBootstrapTest
[junit4] 2> 268050 INFO (SUITE-CdcrBootstrapTest-seed#[3D43DFAB320A40C4]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_3D43DFAB320A40C4-001/init-core-data-001
[junit4] 2> 268050 WARN (SUITE-CdcrBootstrapTest-seed#[3D43DFAB320A40C4]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 268050 INFO (SUITE-CdcrBootstrapTest-seed#[3D43DFAB320A40C4]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 268072 INFO (SUITE-CdcrBootstrapTest-seed#[3D43DFAB320A40C4]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 268104 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testBootstrapWithContinousIndexingOnSourceCluster
[junit4] 2> 268105 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_3D43DFAB320A40C4-001/cdcr-target-001
[junit4] 2> 268106 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 268124 INFO (Thread-73) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 268124 INFO (Thread-73) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 268197 ERROR (Thread-73) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 268421 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.ZkTestServer start zk server on port:60334
[junit4] 2> 268532 INFO (jetty-launcher-60-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 268701 INFO (jetty-launcher-60-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1075f419{/solr,null,AVAILABLE}
[junit4] 2> 269676 INFO (jetty-launcher-60-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4046fad4{SSL,[ssl, http/1.1]}{127.0.0.1:37550}
[junit4] 2> 269676 INFO (jetty-launcher-60-thread-1) [ ] o.e.j.s.Server Started @288801ms
[junit4] 2> 269676 INFO (jetty-launcher-60-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37550}
[junit4] 2> 269676 ERROR (jetty-launcher-60-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 269676 INFO (jetty-launcher-60-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 269677 INFO (jetty-launcher-60-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 269677 INFO (jetty-launcher-60-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 269677 INFO (jetty-launcher-60-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-20T22:31:23.523Z
[junit4] 2> 269928 INFO (jetty-launcher-60-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 269978 INFO (jetty-launcher-60-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60334/solr
[junit4] 2> 270935 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 270936 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37550_solr
[junit4] 2> 270955 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.c.Overseer Overseer (id=98863591063945219-127.0.0.1:37550_solr-n_0000000000) starting
[junit4] 2> 271128 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37550_solr
[junit4] 2> 271154 INFO (OverseerStateUpdate-98863591063945219-127.0.0.1:37550_solr-n_0000000000) [n:127.0.0.1:37550_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 273473 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37550.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e79c86e
[junit4] 2> 273535 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37550.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e79c86e
[junit4] 2> 273536 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37550.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e79c86e
[junit4] 2> 273576 INFO (jetty-launcher-60-thread-1) [n:127.0.0.1:37550_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_3D43DFAB320A40C4-001/cdcr-target-001/node1/.
[junit4] 1> Target zkHost = 127.0.0.1:60334/solr
[junit4] 2> 274166 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_3D43DFAB320A40C4-001/cdcr-source-001
[junit4] 2> 274166 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 274184 INFO (Thread-83) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 274184 INFO (Thread-83) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 274264 ERROR (Thread-83) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 274437 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.ZkTestServer start zk server on port:59846
[junit4] 2> 274522 INFO (jetty-launcher-73-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 274605 INFO (jetty-launcher-73-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@21592feb{/solr,null,AVAILABLE}
[junit4] 2> 274606 INFO (jetty-launcher-73-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@24a4020f{SSL,[ssl, http/1.1]}{127.0.0.1:60685}
[junit4] 2> 274606 INFO (jetty-launcher-73-thread-1) [ ] o.e.j.s.Server Started @293731ms
[junit4] 2> 274606 INFO (jetty-launcher-73-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60685}
[junit4] 2> 274606 ERROR (jetty-launcher-73-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 274684 INFO (jetty-launcher-73-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 274684 INFO (jetty-launcher-73-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 274684 INFO (jetty-launcher-73-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 274684 INFO (jetty-launcher-73-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-20T22:31:28.530Z
[junit4] 2> 274724 INFO (jetty-launcher-73-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 275156 INFO (jetty-launcher-73-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59846/solr
[junit4] 2> 277452 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 277452 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60685_solr
[junit4] 2> 277453 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.c.Overseer Overseer (id=98863591460044803-127.0.0.1:60685_solr-n_0000000000) starting
[junit4] 2> 277584 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60685_solr
[junit4] 2> 277623 INFO (zkCallback-80-thread-1-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 278718 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60685.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e79c86e
[junit4] 2> 278925 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60685.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e79c86e
[junit4] 2> 278925 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60685.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e79c86e
[junit4] 2> 278926 INFO (jetty-launcher-73-thread-1) [n:127.0.0.1:60685_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_3D43DFAB320A40C4-001/cdcr-source-001/node1/.
[junit4] 2> 279957 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 279958 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59846/solr ready
[junit4] 2> 280513 INFO (qtp183448557-392) [n:127.0.0.1:60685_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=cdcr-source&name=cdcr-source&nrtReplicas=1&action=CREATE&numShards=1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 280523 INFO (OverseerThreadFactory-145-thread-1-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.CreateCollectionCmd Create collection cdcr-source
[junit4] 2> 280846 INFO (OverseerStateUpdate-98863591460044803-127.0.0.1:60685_solr-n_0000000000) [n:127.0.0.1:60685_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-source",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"cdcr-source_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:60685/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 281073 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=cdcr-source&newCollection=true&collection=cdcr-source&version=2&replicaType=NRT&coreNodeName=core_node2&name=cdcr-source_shard1_replica_n1&action=CREATE&numShards=1&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
[junit4] 2> 281074 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 281249 INFO (zkCallback-80-thread-1-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 281269 INFO (zkCallback-80-thread-2-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 282374 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 282471 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
[junit4] 2> 282495 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.IndexSchema [cdcr-source_shard1_replica_n1] Schema name=minimal
[junit4] 2> 282569 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 282920 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Created and persisted managed schema znode at /configs/cdcr-source/managed-schema
[junit4] 2> 282924 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, renamed the non-managed schema /configs/cdcr-source/schema.xml to /configs/cdcr-source/schema.xml.bak
[junit4] 2> 282924 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-source_shard1_replica_n1' using configuration from collection cdcr-source, trusted=true
[junit4] 2> 282925 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60685.solr.core.cdcr-source.shard1.replica_n1' (registry 'solr.core.cdcr-source.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e79c86e
[junit4] 2> 282925 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 282925 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [[cdcr-source_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_3D43DFAB320A40C4-001/cdcr-source-001/node1/cdcr-source_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_3D43DFAB320A40C4-001/cdcr-source-001/node1/./cdcr-source_shard1_replica_n1/data/]
[junit4] 2> 284148 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 284148 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 284150 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 284150 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 284260 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@510baeec[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 284261 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-source
[junit4] 2> 284262 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-source
[junit4] 2> 284264 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-source/managed-schema
[junit4] 2> 284282 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 284283 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 284400 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrBufferStateManager Created znode /collections/cdcr-source/cdcr/state/buffer
[junit4] 2> 284413 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.h.CdcrProcessStateManager Created znode /collections/cdcr-source/cdcr/state/process
[junit4] 2> 285240 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581817474932801536
[junit4] 2> 285297 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@510baeec[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 285301 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 285301 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 285301 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:60685/solr/cdcr-source_shard1_replica_n1/
[junit4] 2> 285301 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 285301 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:60685/solr/cdcr-source_shard1_replica_n1/ has no replicas
[junit4] 2> 285301 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 285333 INFO (zkCallback-80-thread-3-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-source:shard1
[junit4] 2> 285353 INFO (zkCallback-80-thread-2-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 285353 INFO (zkCallback-80-thread-3-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 285355 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:60685/solr/cdcr-source_shard1_replica_n1/ shard1
[junit4] 2> 285355 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 285366 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=cdcr-source&newCollection=true&collection=cdcr-source&version=2&replicaType=NRT&coreNodeName=core_node2&name=cdcr-source_shard1_replica_n1&action=CREATE&numShards=1&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=4292
[junit4] 2> 285405 INFO (qtp183448557-392) [n:127.0.0.1:60685_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 285485 INFO (zkCallback-80-thread-2-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 285485 INFO (zkCallback-80-thread-3-processing-n:127.0.0.1:60685_solr) [n:127.0.0.1:60685_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 286428 INFO (qtp183448557-392) [n:127.0.0.1:60685_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=cdcr-source&name=cdcr-source&nrtReplicas=1&action=CREATE&numShards=1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2} status=0 QTime=5915
[junit4] 1> Adding 10 docs with commit=true, numDocs=100
[junit4] 2> 286549 INFO (OverseerCollectionConfigSetProcessor-98863591460044803-127.0.0.1:60685_solr-n_0000000000) [n:127.0.0.1:60685_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 287011 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=494
[junit4] 2> 287064 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817476845404160,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 287065 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817476845404160
[junit4] 2> 287544 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4422f037[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 287546 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@4422f037[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100)))}
[junit4] 2> 287547 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 287547 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=482
[junit4] 1> Adding 10 docs with commit=true, numDocs=200
[junit4] 2> 287778 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=229
[junit4] 2> 287800 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817477617156096,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 287800 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817477617156096
[junit4] 2> 288412 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@44c22b12[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 288413 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 288414 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@44c22b12[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100)))}
[junit4] 2> 288415 INFO (qtp183448557-395) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=615
[junit4] 1> Adding 10 docs with commit=true, numDocs=300
[junit4] 2> 288684 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=266
[junit4] 2> 288685 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817478545145856,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 288744 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817478545145856
[junit4] 2> 289112 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@404f72e6[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 289114 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@404f72e6[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100)))}
[junit4] 2> 289114 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 289115 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=429
[junit4] 1> Adding 10 docs with commit=true, numDocs=400
[junit4] 2> 289357 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=239
[junit4] 2> 289385 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817479279149056,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 289385 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817479279149056
[junit4] 2> 289684 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7056aaf3[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 289685 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 289686 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@7056aaf3[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100) Uninverting(_3(7.2.0):C100)))}
[junit4] 2> 289687 INFO (qtp183448557-397) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=301
[junit4] 1> Adding 10 docs with commit=true, numDocs=500
[junit4] 2> 289790 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=101
[junit4] 2> 289805 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817479719550976,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 289805 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817479719550976
[junit4] 2> 290089 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5d1d5a6d[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 290089 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 290091 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@5d1d5a6d[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100) Uninverting(_3(7.2.0):C100) Uninverting(_4(7.2.0):C100)))}
[junit4] 2> 290091 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=286
[junit4] 1> Adding 10 docs with commit=true, numDocs=600
[junit4] 2> 290184 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=91
[junit4] 2> 290220 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817480154710016,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 290221 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817480154710016
[junit4] 2> 292633 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@49e4bdd5[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 292633 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 292635 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@49e4bdd5[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100) Uninverting(_3(7.2.0):C100) Uninverting(_4(7.2.0):C100) Uninverting(_5(7.2.0):C100)))}
[junit4] 2> 292635 INFO (qtp183448557-393) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=2415
[junit4] 1> Adding 10 docs with commit=true, numDocs=700
[junit4] 2> 292674 INFO (qtp183448557-396) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=37
[junit4] 2> 292704 INFO (qtp183448557-396) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817482759372800,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 292704 INFO (qtp183448557-396) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817482759372800
[junit4] 2> 293037 INFO (qtp183448557-396) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3f062990[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 293037 INFO (qtp183448557-396) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 293039 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@3f062990[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100) Uninverting(_3(7.2.0):C100) Uninverting(_4(7.2.0):C100) Uninverting(_5(7.2.0):C100) Uninverting(_6(7.2.0):C100)))}
[junit4] 2> 293039 INFO (qtp183448557-396) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=334
[junit4] 1> Adding 10 docs with commit=true, numDocs=800
[junit4] 2> 293125 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=68
[junit4] 2> 293126 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817483201871872,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 293126 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817483201871872
[junit4] 2> 293442 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@468a5fd[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 293444 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@468a5fd[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100) Uninverting(_3(7.2.0):C100) Uninverting(_4(7.2.0):C100) Uninverting(_5(7.2.0):C100) Uninverting(_6(7.2.0):C100) Uninverting(_7(7.2.0):C100)))}
[junit4] 2> 293445 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 293445 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=318
[junit4] 1> Adding 10 docs with commit=true, numDocs=900
[junit4] 2> 293469 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=22
[junit4] 2> 293471 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817483563630592,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 293471 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817483563630592
[junit4] 2> 293720 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@76d345f4[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 293723 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@76d345f4[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100) Uninverting(_3(7.2.0):C100) Uninverting(_4(7.2.0):C100) Uninverting(_5(7.2.0):C100) Uninverting(_6(7.2.0):C100) Uninverting(_7(7.2.0):C100) Uninverting(_8(7.2.0):C100)))}
[junit4] 2> 293723 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 293723 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=252
[junit4] 1> Adding 10 docs with commit=true, numDocs=1000
[junit4] 2> 293755 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&wt=javabin&version=2} status=0 QTime=29
[junit4] 2> 293800 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581817483908612096,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 293800 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@16d636e5 commitCommandVersion:1581817483908612096
[junit4] 2> 294168 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3be019f2[cdcr-source_shard1_replica_n1] main]
[junit4] 2> 294170 INFO (searcherExecutor-148-thread-1-processing-n:127.0.0.1:60685_solr x:cdcr-source_shard1_replica_n1 s:shard1 c:cdcr-source r:core_node2) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-source_shard1_replica_n1] Registered new searcher Searcher@3be019f2[cdcr-source_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C100) Uninverting(_1(7.2.0):C100) Uninverting(_2(7.2.0):C100) Uninverting(_3(7.2.0):C100) Uninverting(_4(7.2.0):C100) Uninverting(_5(7.2.0):C100) Uninverting(_6(7.2.0):C100) Uninverting(_7(7.2.0):C100) Uninverting(_8(7.2.0):C100) Uninverting(_9(7.2.0):C100)))}
[junit4] 2> 294170 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 294171 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-source:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=370
[junit4] 2> 294173 INFO (qtp183448557-398) [n:127.0.0.1:60685_solr c:cdcr-source s:shard1 r:core_node2 x:cdcr-source_shard1_replica_n1] o.a.s.c.S.Request [cdcr-source_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&_stateVer_=cdcr-source:4&wt=javabin&version=2} hits=1000 status=0 QTime=0
[junit4] 2> 295013 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 295014 INFO (TEST-CdcrBootstrapTest.testBootstrapWithContinousIndexingOnSourceCluster-seed#[3D43DFAB320A40C4]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60334/solr ready
[junit4] 2> 296302 INFO (qtp547630103-352) [n:127.0.0.1:37550_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=cdcr-target&name=cdcr-target&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 296322 INFO (OverseerThreadFactory-135-thread-1-processing-n:127.0.0.1:37550_solr) [n:127.0.0.1:37550_solr ] o.a.s.c.CreateCollectionCmd Create collection cdcr-target
[junit4] 2> 296516 INFO (OverseerStateUpdate-98863591063945219-127.0.0.1:37550_solr-n_0000000000) [n:127.0.0.1:37550_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-target",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"cdcr-target_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:37550/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 297028 INFO (qtp547630103-349) [n:127.0.0.1:37550_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=cdcr-target&newCollection=true&name=cdcr-target_shard1_replica_n1&action=CREATE&numShards=1&collection=cdcr-target&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 297029 INFO (qtp547630103-349) [n:127.0.0.1:37550_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 297293 INFO (zkCallback-67-thread-1-processing-n:127.0.0.1:37550_solr) [n:127.0.0.1:37550_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 297333 INFO (zkCallback-67-thread-2-processing-n:127.0.0.1:37550_solr) [n:127.0.0.1:37550_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 298214 INFO (qtp547630103-349) [n:127.0.0.1:37550_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 298272 INFO (qtp547630103-349) [n:127.0.0.1:37550_solr c:cdcr-target s:shard1 r:core_node2 x:cdcr-target_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed sche
[...truncated too long message...]
questHandlerTest, SolrCloudExampleTest, HighlighterConfigTest, DocValuesMultiTest, DeleteLastCustomShardedReplicaTest, TestDelegationWithHadoopAuth, TestCustomStream, TestSearcherReuse, TestConfigSetsAPIZkFailure, TestDefaultStatsCache, TestPerFieldSimilarity, HdfsChaosMonkeySafeLeaderTest, VMParamsZkACLAndCredentialsProvidersTest, SuggesterTSTTest, ClassificationUpdateProcessorFactoryTest, DebugComponentTest, TestDocTermOrds, OpenCloseCoreStressTest, CollectionTooManyReplicasTest, OutputWriterTest, BadIndexSchemaTest, HdfsBasicDistributedZk2Test, TestFastWriter, CloudExitableDirectoryReaderTest, SolrIndexConfigTest, SchemaVersionSpecificBehaviorTest, TestAnalyzedSuggestions, TestExceedMaxTermLength, HdfsUnloadDistributedZkTest, NotRequiredUniqueKeyTest, SystemLogListenerTest, MergeStrategyTest, SpatialRPTFieldTypeTest, TestSolrCoreSnapshots, MBeansHandlerTest, MigrateRouteKeyTest, TestSegmentSorting, TestBackupRepositoryFactory, BinaryUpdateRequestHandlerTest, SharedFSAutoReplicaFailoverTest, TestCollectionAPIs, BufferStoreTest, TestMiniSolrCloudClusterSSL]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestMiniSolrCloudClusterSSL -Dtests.seed=3D43DFAB320A40C4 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es-MX -Dtests.timezone=Pacific/Port_Moresby -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.00s J1 | TestMiniSolrCloudClusterSSL (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.TestMiniSolrCloudClusterSSL:
[junit4] > 1) Thread[id=9145, name=OverseerAutoScalingTriggerThread-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=WAITING, group=Overseer autoscaling triggers]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:161)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 2) Thread[id=9053, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 3) Thread[id=9146, name=ScheduledTrigger-4119-thread-1, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:389)
[junit4] > at org.apache.solr.cloud.ZkDistributedQueue.offer(ZkDistributedQueue.java:293)
[junit4] > at org.apache.solr.cloud.autoscaling.TriggerEventQueue.offerEvent(TriggerEventQueue.java:58)
[junit4] > at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.enqueue(ScheduledTriggers.java:411)
[junit4] > at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:205)
[junit4] > at org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$237/524708820.process(Unknown Source)
[junit4] > at org.apache.solr.cloud.autoscaling.NodeLostTrigger.run(NodeLostTrigger.java:163)
[junit4] > at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:453)
[junit4] > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] > at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 4) Thread[id=9076, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=9143, name=OverseerStateUpdate-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=TIMED_WAITING, group=Overseer state updater.]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435)
[junit4] > at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:202)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=9144, name=OverseerCollectionConfigSetProcessor-98863861775663115-127.0.0.1:55856_solr-n_0000000003, state=TIMED_WAITING, group=Overseer collection creation process.]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
[junit4] > at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)
[junit4] > at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=9108, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-EventThread, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
[junit4] > 8) Thread[id=9054, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 9) Thread[id=9107, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-SendThread(127.0.0.1:48343), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
[junit4] > at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=9146, name=ScheduledTrigger-4119-thread-1, state=WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 2) Thread[id=9107, name=zkCallback-1345-thread-2-processing-n:127.0.0.1:55856_solr-SendThread(127.0.0.1:48343), state=TIMED_WAITING, group=TGRP-TestMiniSolrCloudClusterSSL]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
[junit4] > at __randomizedtesting.SeedInfo.seed([3D43DFAB320A40C4]:0)
[junit4] Completed [310/744 (3!)] on J1 in 255.65s, 4 tests, 1 failure, 2 errors <<< FAILURES!
[...truncated 48579 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 188 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/188/
7 tests failed.
FAILED: org.apache.solr.cloud.HttpPartitionTest.test
Error Message:
Could not load collection from ZK: c8n_1x2
Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: c8n_1x2
at __randomizedtesting.SeedInfo.seed([5D3C760E1FD07E28:D56849D4B12C13D0]:0)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1963)
at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:370)
at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/c8n_1x2/state.json
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
... 46 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestCloudRecovery
Error Message:
Error from server at https://127.0.0.1:34391/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000000
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:34391/solr: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000000
at __randomizedtesting.SeedInfo.seed([5D3C760E1FD07E28]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.TestCloudRecovery.setupCluster(TestCloudRecovery.java:68)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.SearchRateTriggerTest.testTrigger
Error Message:
expected:<0> but was:<1>
Stack Trace:
java.lang.AssertionError: expected:<0> but was:<1>
at __randomizedtesting.SeedInfo.seed([5D3C760E1FD07E28:3EF7408C861F0D05]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.autoscaling.SearchRateTriggerTest.testTrigger(SearchRateTriggerTest.java:101)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.update.processor.TestNamedUpdateProcessors.test
Error Message:
{ "responseHeader":{ "status":500, "QTime":31981}, "errorMessages":["4 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [https://127.0.0.1:53437/collection1_shard2_replica_n41/, https://127.0.0.1:44108/collection1_shard2_replica_n45/, https://127.0.0.1:59937/collection1_shard1_replica_n43/, https://127.0.0.1:36887/collection1_shard1_replica_n47/]\n"], "WARNING":"This response format is experimental. It is likely to change in the future.", "error":{ "metadata":[ "error-class","org.apache.solr.common.SolrException", "root-error-class","org.apache.solr.common.SolrException"], "msg":"4 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [https://127.0.0.1:53437/collection1_shard2_replica_n41/, https://127.0.0.1:44108/collection1_shard2_replica_n45/, https://127.0.0.1:59937/collection1_shard1_replica_n43/, https://127.0.0.1:36887/collection1_shard1_replica_n47/]", "trace":"org.apache.solr.common.SolrException: 4 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [https://127.0.0.1:53437/collection1_shard2_replica_n41/, https://127.0.0.1:44108/collection1_shard2_replica_n45/, https://127.0.0.1:59937/collection1_shard1_replica_n43/, https://127.0.0.1:36887/collection1_shard1_replica_n47/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:772)\n\tat org.apache.solr.handler.SolrConfigHandler.access$400(SolrConfigHandler.java:102)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:503)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:346)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.access$100(SolrConfigHandler.java:159)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2484)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:710)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:516)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:534)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)\n\tat java.lang.Thread.run(Thread.java:748)\n", "code":500}}
Stack Trace:
java.lang.AssertionError: {
"responseHeader":{
"status":500,
"QTime":31981},
"errorMessages":["4 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [https://127.0.0.1:53437/collection1_shard2_replica_n41/, https://127.0.0.1:44108/collection1_shard2_replica_n45/, https://127.0.0.1:59937/collection1_shard1_replica_n43/, https://127.0.0.1:36887/collection1_shard1_replica_n47/]\n"],
"WARNING":"This response format is experimental. It is likely to change in the future.",
"error":{
"metadata":[
"error-class","org.apache.solr.common.SolrException",
"root-error-class","org.apache.solr.common.SolrException"],
"msg":"4 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [https://127.0.0.1:53437/collection1_shard2_replica_n41/, https://127.0.0.1:44108/collection1_shard2_replica_n45/, https://127.0.0.1:59937/collection1_shard1_replica_n43/, https://127.0.0.1:36887/collection1_shard1_replica_n47/]",
"trace":"org.apache.solr.common.SolrException: 4 out of 5 the property overlay to be of version 0 within 30 seconds! Failed cores: [https://127.0.0.1:53437/collection1_shard2_replica_n41/, https://127.0.0.1:44108/collection1_shard2_replica_n45/, https://127.0.0.1:59937/collection1_shard1_replica_n43/, https://127.0.0.1:36887/collection1_shard1_replica_n47/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:772)\n\tat org.apache.solr.handler.SolrConfigHandler.access$400(SolrConfigHandler.java:102)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:503)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:346)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.access$100(SolrConfigHandler.java:159)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2484)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:710)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:516)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:534)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)\n\tat java.lang.Thread.run(Thread.java:748)\n",
"code":500}}
at __randomizedtesting.SeedInfo.seed([5D3C760E1FD07E28:D56849D4B12C13D0]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.core.TestSolrConfigHandler.runConfigCommand(TestSolrConfigHandler.java:185)
at org.apache.solr.update.processor.TestNamedUpdateProcessors.test(TestNamedUpdateProcessors.java:85)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI
Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:52419/solr/awhollynewcollection_0, http://127.0.0.1:59254/solr/awhollynewcollection_0, http://127.0.0.1:59800/solr/awhollynewcollection_0, http://127.0.0.1:45558/solr/awhollynewcollection_0]
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:52419/solr/awhollynewcollection_0, http://127.0.0.1:59254/solr/awhollynewcollection_0, http://127.0.0.1:59800/solr/awhollynewcollection_0, http://127.0.0.1:45558/solr/awhollynewcollection_0]
at __randomizedtesting.SeedInfo.seed([5D3C760E1FD07E28:154902BA19E351BD]:0)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:867)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:800)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:460)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59254/solr/awhollynewcollection_0: no servers hosting shard: shard5
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
... 45 more
FAILED: org.apache.solr.schema.TestCloudSchemaless.test
Error Message:
Error from server at http://127.0.0.1:43259/jglt/b/collection1: 4 Async exceptions during distributed update: Error from server at http://127.0.0.1:60536/jglt/b/collection1_shard1_replica_n87: Server Error request: http://127.0.0.1:60536/jglt/b/collection1_shard1_replica_n87/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2 Remote error message: Error persisting managed schema at /configs/conf1/managed-schema Error from server at http://127.0.0.1:43224/jglt/b/collection1_shard4_replica_n85: Server Error request: http://127.0.0.1:43224/jglt/b/collection1_shard4_replica_n85/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2 Remote error message: Error persisting managed schema at /configs/conf1/managed-schema Error from server at http://127.0.0.1:37445/jglt/b/collection1_shard3_replica_n83: Server Error request: http://127.0.0.1:37445/jglt/b/collection1_shard3_replica_n83/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2 Remote error message: Error persisting managed schema at /configs/conf1/managed-schema Error from server at http://127.0.0.1:51945/jglt/b/collection1_shard2_replica_n81: Server Error request: http://127.0.0.1:51945/jglt/b/collection1_shard2_replica_n81/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2 Remote error message: Error persisting managed schema at /configs/conf1/managed-schema
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:43259/jglt/b/collection1: 4 Async exceptions during distributed update:
Error from server at http://127.0.0.1:60536/jglt/b/collection1_shard1_replica_n87: Server Error
request: http://127.0.0.1:60536/jglt/b/collection1_shard1_replica_n87/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2
Remote error message: Error persisting managed schema at /configs/conf1/managed-schema
Error from server at http://127.0.0.1:43224/jglt/b/collection1_shard4_replica_n85: Server Error
request: http://127.0.0.1:43224/jglt/b/collection1_shard4_replica_n85/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2
Remote error message: Error persisting managed schema at /configs/conf1/managed-schema
Error from server at http://127.0.0.1:37445/jglt/b/collection1_shard3_replica_n83: Server Error
request: http://127.0.0.1:37445/jglt/b/collection1_shard3_replica_n83/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2
Remote error message: Error persisting managed schema at /configs/conf1/managed-schema
Error from server at http://127.0.0.1:51945/jglt/b/collection1_shard2_replica_n81: Server Error
request: http://127.0.0.1:51945/jglt/b/collection1_shard2_replica_n81/update?update.chain=add-unknown-fields-to-the-schema&update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A43259%2Fjglt%2Fb%2Fcollection1_shard3_replica_n91%2F&wt=javabin&version=2
Remote error message: Error persisting managed schema at /configs/conf1/managed-schema
at __randomizedtesting.SeedInfo.seed([5D3C760E1FD07E28:D56849D4B12C13D0]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:106)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:71)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:85)
at org.apache.solr.schema.TestCloudSchemaless.test(TestCloudSchemaless.java:124)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit
Error Message:
Shard split did not succeed after a previous failed split attempt left sub-shards in construction state
Stack Trace:
java.lang.AssertionError: Shard split did not succeed after a previous failed split attempt left sub-shards in construction state
at __randomizedtesting.SeedInfo.seed([5D3C760E1FD07E28:A471E5A123A533A2]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:294)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12238 lines...]
[junit4] Suite: org.apache.solr.cloud.TestCloudRecovery
[junit4] 2> 2538711 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/init-core-data-001
[junit4] 2> 2538712 WARN (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=21 numCloses=21
[junit4] 2> 2538712 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2538714 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 2538723 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001
[junit4] 2> 2538724 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2539001 INFO (Thread-1492) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2539001 INFO (Thread-1492) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2539114 ERROR (Thread-1492) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2539114 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:41174
[junit4] 2> 2539502 INFO (jetty-launcher-838-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2539513 INFO (jetty-launcher-838-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2539514 INFO (jetty-launcher-838-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@45f94c23{/solr,null,AVAILABLE}
[junit4] 2> 2539535 INFO (jetty-launcher-838-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1c94313d{SSL,[ssl, http/1.1]}{127.0.0.1:34391}
[junit4] 2> 2539535 INFO (jetty-launcher-838-thread-2) [ ] o.e.j.s.Server Started @2557456ms
[junit4] 2> 2539535 INFO (jetty-launcher-838-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34391}
[junit4] 2> 2539535 ERROR (jetty-launcher-838-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2539535 INFO (jetty-launcher-838-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 2539535 INFO (jetty-launcher-838-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2539535 INFO (jetty-launcher-838-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2539535 INFO (jetty-launcher-838-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-20T08:00:30.166Z
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3648a272{/solr,null,AVAILABLE}
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@69ceb7c8{SSL,[ssl, http/1.1]}{127.0.0.1:47185}
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.e.j.s.Server Started @2557459ms
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=47185}
[junit4] 2> 2539538 ERROR (jetty-launcher-838-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2539538 INFO (jetty-launcher-838-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-20T08:00:30.169Z
[junit4] 2> 2539558 INFO (jetty-launcher-838-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2539632 INFO (jetty-launcher-838-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2539922 INFO (jetty-launcher-838-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41174/solr
[junit4] 2> 2540624 INFO (jetty-launcher-838-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41174/solr
[junit4] 2> 2540910 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2540910 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:47185_solr
[junit4] 2> 2540911 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.c.Overseer Overseer (id=98860166652624902-127.0.0.1:47185_solr-n_0000000000) starting
[junit4] 2> 2541023 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47185_solr
[junit4] 2> 2541177 INFO (OverseerStateUpdate-98860166652624902-127.0.0.1:47185_solr-n_0000000000) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2541910 INFO (jetty-launcher-838-thread-2) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2541911 INFO (jetty-launcher-838-thread-2) [n:127.0.0.1:34391_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2541942 INFO (jetty-launcher-838-thread-2) [n:127.0.0.1:34391_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34391_solr
[junit4] 2> 2541943 INFO (zkCallback-851-thread-1-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2542250 INFO (zkCallback-849-thread-1-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2543120 INFO (jetty-launcher-838-thread-2) [n:127.0.0.1:34391_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34391.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2543245 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47185.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2543354 INFO (jetty-launcher-838-thread-2) [n:127.0.0.1:34391_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34391.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2543451 INFO (jetty-launcher-838-thread-2) [n:127.0.0.1:34391_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34391.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2543452 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47185.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2543452 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47185.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2543467 INFO (jetty-launcher-838-thread-2) [n:127.0.0.1:34391_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node2/.
[junit4] 2> 2543467 INFO (jetty-launcher-838-thread-1) [n:127.0.0.1:47185_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node1/.
[junit4] 2> 2544082 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2544082 INFO (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41174/solr ready
[junit4] 2> 2544378 INFO (qtp698573975-5431) [n:127.0.0.1:34391_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=0&collection.configName=config&maxShardsPerNode=2&name=collection1&nrtReplicas=0&action=CREATE&numShards=2&tlogReplicas=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2544433 INFO (OverseerThreadFactory-2191-thread-1-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 2544647 INFO (OverseerStateUpdate-98860166652624902-127.0.0.1:47185_solr-n_0000000000) [n:127.0.0.1:47185_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"collection1_shard1_replica_t1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:34391/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2544650 INFO (OverseerStateUpdate-98860166652624902-127.0.0.1:47185_solr-n_0000000000) [n:127.0.0.1:47185_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"collection1_shard1_replica_t3",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:47185/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2544670 INFO (OverseerStateUpdate-98860166652624902-127.0.0.1:47185_solr-n_0000000000) [n:127.0.0.1:47185_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"collection1_shard2_replica_t5",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:34391/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2544672 INFO (OverseerStateUpdate-98860166652624902-127.0.0.1:47185_solr-n_0000000000) [n:127.0.0.1:47185_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"collection1_shard2_replica_t7",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:47185/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2545200 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 2545201 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2545235 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t7&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 2545235 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2545816 INFO (zkCallback-849-thread-1-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2545816 INFO (zkCallback-851-thread-1-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2545819 INFO (zkCallback-851-thread-2-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2545856 INFO (zkCallback-849-thread-2-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2545902 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=collection1_shard1_replica_t3&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 2545922 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node6&collection.configName=config&newCollection=true&name=collection1_shard2_replica_t5&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 2546085 INFO (zkCallback-849-thread-2-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2546085 INFO (zkCallback-849-thread-1-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2546086 INFO (zkCallback-851-thread-3-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2546086 INFO (zkCallback-851-thread-2-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2546777 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2546812 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2565560 WARN (jetty-launcher-838-thread-2-SendThread(127.0.0.1:41174)) [n:127.0.0.1:34391_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 18626ms for sessionid 0x15f38cc3bc40005
[junit4] 2> 2565568 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.s.IndexSchema [collection1_shard2_replica_t7] Schema name=minimal
[junit4] 2> 2565589 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2565589 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t7' using configuration from collection collection1, trusted=true
[junit4] 2> 2565590 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47185.solr.core.collection1.shard2.replica_t7' (registry 'solr.core.collection1.shard2.replica_t7') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2565590 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2565590 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.c.SolrCore [[collection1_shard2_replica_t7] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node1/collection1_shard2_replica_t7], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node1/./collection1_shard2_replica_t7/data/]
[junit4] 2> 2565560 WARN (jetty-launcher-838-thread-1-SendThread(127.0.0.1:41174)) [n:127.0.0.1:47185_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 18636ms for sessionid 0x15f38cc3bc40006
[junit4] 2> 2565765 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f38cc3bc40006, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2565794 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f38cc3bc40005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2565678 WARN (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3187160e name: ZooKeeperConnection Watcher:127.0.0.1:41174/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 2565561 WARN (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker-SendThread(127.0.0.1:41174)) [ ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 21119ms for sessionid 0x15f38cc3bc40008
[junit4] 2> 2565794 WARN (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 2565918 WARN (zkCallback-851-thread-2-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@103cfaa5 name: ZooKeeperConnection Watcher:127.0.0.1:41174/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 2565918 WARN (zkCallback-851-thread-2-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 2565921 WARN (zkCallback-857-thread-2) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7e307391 name: ZooKeeperConnection Watcher:127.0.0.1:41174/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 2565921 WARN (zkCallback-857-thread-2) [ ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 2566357 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2566357 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2566359 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2566359 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2566397 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.s.SolrIndexSearcher Opening [Searcher@538b0dd1[collection1_shard2_replica_t7] main]
[junit4] 2> 2567036 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
[junit4] 2> 2567037 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
[junit4] 2> 2567038 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2567038 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581762695836729344
[junit4] 2> 2567078 INFO (searcherExecutor-2196-thread-1-processing-n:127.0.0.1:47185_solr x:collection1_shard2_replica_t7 s:shard2 c:collection1 r:core_node8) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.c.SolrCore [collection1_shard2_replica_t7] Registered new searcher Searcher@538b0dd1[collection1_shard2_replica_t7] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2567106 INFO (qtp932038018-5434) [n:127.0.0.1:47185_solr c:collection1 s:shard2 r:core_node8 x:collection1_shard2_replica_t7] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=9999ms
[junit4] 2> 2567129 INFO (zkCallback-851-thread-8-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2567141 INFO (zkCallback-851-thread-7-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2567397 INFO (zkCallback-851-thread-7-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 2567410 WARN (jetty-launcher-838-thread-2-SendThread(127.0.0.1:41174)) [n:127.0.0.1:34391_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15f38cc3bc40005 has expired
[junit4] 2> 2567413 WARN (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3187160e name: ZooKeeperConnection Watcher:127.0.0.1:41174/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 2567413 WARN (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 2567413 WARN (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 2567425 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 2567425 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
[junit4] 2> 2567457 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 2567463 WARN (SUITE-TestCloudRecovery-seed#[5D3C760E1FD07E28]-worker-SendThread(127.0.0.1:41174)) [ ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15f38cc3bc40008 has expired
[junit4] 2> 2567464 WARN (zkCallback-857-thread-4) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7e307391 name: ZooKeeperConnection Watcher:127.0.0.1:41174/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 2567464 WARN (zkCallback-857-thread-4) [ ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 2567464 WARN (zkCallback-857-thread-4) [ ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 2567502 INFO (zkCallback-857-thread-4) [ ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 2567517 ERROR (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.c.CloudConfigSetService null:org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
[junit4] 2> at org.apache.solr.cloud.CloudConfigSetService.createCoreResourceLoader(CloudConfigSetService.java:44)
[junit4] 2> at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:75)
[junit4] 2> at org.apache.solr.core.CoreContainer.getConfigSet(CoreContainer.java:1080)
[junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1032)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:948)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:497)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2>
[junit4] 2> 2567527 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2567569 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34391_solr
[junit4] 2> 2567593 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper
[junit4] 2> 2567593 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ConnectionManager Connected:true
[junit4] 2> 2567612 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2567650 INFO (zkCallback-851-thread-7-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2567666 INFO (zkCallback-857-thread-4) [ ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper
[junit4] 2> 2567666 INFO (zkCallback-857-thread-4) [ ] o.a.s.c.c.ConnectionManager Connected:true
[junit4] 2> 2567689 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2567695 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.s.IndexSchema [collection1_shard2_replica_t5] Schema name=minimal
[junit4] 2> 2567751 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.s.IndexSchema [collection1_shard1_replica_t3] Schema name=minimal
[junit4] 2> 2567765 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2567765 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t3' using configuration from collection collection1, trusted=true
[junit4] 2> 2567766 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_47185.solr.core.collection1.shard1.replica_t3' (registry 'solr.core.collection1.shard1.replica_t3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2567766 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2567766 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.c.SolrCore [[collection1_shard1_replica_t3] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node1/collection1_shard1_replica_t3], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node1/./collection1_shard1_replica_t3/data/]
[junit4] 2> 2567785 INFO (zkCallback-851-thread-7-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2567786 INFO (zkCallback-851-thread-8-processing-n:127.0.0.1:47185_solr) [n:127.0.0.1:47185_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2567786 INFO (zkCallback-849-thread-4-processing-n:127.0.0.1:34391_solr) [n:127.0.0.1:34391_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2567787 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2567787 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t5' using configuration from collection collection1, trusted=true
[junit4] 2> 2567788 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34391.solr.core.collection1.shard2.replica_t5' (registry 'solr.core.collection1.shard2.replica_t5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2567788 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2567788 INFO (qtp698573975-5425) [n:127.0.0.1:34391_solr c:collection1 s:shard2 r:core_node6 x:collection1_shard2_replica_t5] o.a.s.c.SolrCore [[collection1_shard2_replica_t5] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node2/collection1_shard2_replica_t5], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudRecovery_5D3C760E1FD07E28-001/tempDir-001/node2/./collection1_shard2_replica_t5/data/]
[junit4] 2> 2568379 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2568379 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2568386 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2568386 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2568432 INFO (qtp932038018-5441) [n:127.0.0.1:47185_solr c:collection1 s:shard1 r:core_node4 x:collection1_shard1_replica_t3] o.a.s.s.SolrIndexSearcher Opening [Searcher@3c9e875e[collection1_shard1_replica_t3] main]
[junit4] 2> 2568556 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.s.IndexSchema [collection1_shard1_replica_t1] Schema name=minimal
[junit4] 2> 2568580 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2568580 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t1' using configuration from collection collection1, trusted=true
[junit4] 2> 2568581 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34391.solr.core.collection1.shard1.replica_t1' (registry 'solr.core.collection1.shard1.replica_t1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a17f0a4
[junit4] 2> 2568581 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica_t1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2568593 INFO (qtp698573975-5430) [n:127.0.0.1:34391_solr c:collection1 s:shard1 r:core_node2 x:collec
[...truncated too long message...]
2a3c387: rootName = solr_57520, domain = solr.core.SearchRateTriggerTest-collection1.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.SearchRateTriggerTest-collection1.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@18936c02
[junit4] 2> 9392383 INFO (coreCloseExecutor-8765-thread-2) [n:127.0.0.1:57520_solr c:SearchRateTriggerTest-collection2 s:shard2 r:core_node6 x:SearchRateTriggerTest-collection2_shard2_replica_n5] o.a.s.c.SolrCore [SearchRateTriggerTest-collection2_shard2_replica_n5] CLOSING SolrCore org.apache.solr.core.SolrCore@6a79cd06
[junit4] 2> 9392389 INFO (coreCloseExecutor-8765-thread-2) [n:127.0.0.1:57520_solr c:SearchRateTriggerTest-collection2 s:shard2 r:core_node6 x:SearchRateTriggerTest-collection2_shard2_replica_n5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.SearchRateTriggerTest-collection2.shard2.replica_n5, tag=1786367238
[junit4] 2> 9392389 INFO (coreCloseExecutor-8765-thread-2) [n:127.0.0.1:57520_solr c:SearchRateTriggerTest-collection2 s:shard2 r:core_node6 x:SearchRateTriggerTest-collection2_shard2_replica_n5] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4c790c85: rootName = solr_57520, domain = solr.core.SearchRateTriggerTest-collection2.shard2.replica_n5, service url = null, agent id = null] for registry solr.core.SearchRateTriggerTest-collection2.shard2.replica_n5 / com.codahale.metrics.MetricRegistry@36bb7d9f
[junit4] 2> 9392434 INFO (coreCloseExecutor-8765-thread-2) [n:127.0.0.1:57520_solr c:SearchRateTriggerTest-collection2 s:shard2 r:core_node6 x:SearchRateTriggerTest-collection2_shard2_replica_n5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.SearchRateTriggerTest-collection2.shard2.leader, tag=1786367238
[junit4] 2> 9392482 INFO (coreCloseExecutor-8765-thread-1) [n:127.0.0.1:57520_solr c:SearchRateTriggerTest-collection1 s:shard1 r:core_node5 x:SearchRateTriggerTest-collection1_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.SearchRateTriggerTest-collection1.shard1.leader, tag=30065930
[junit4] 2> 9392803 INFO (jetty-closer-4996-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@193ce9af{/solr,null,UNAVAILABLE}
[junit4] 2> 9392804 INFO (jetty-closer-4996-thread-2) [ ] o.a.s.c.Overseer Overseer (id=98860612339105800-127.0.0.1:54907_solr-n_0000000000) closing
[junit4] 2> 9392804 INFO (OverseerStateUpdate-98860612339105800-127.0.0.1:54907_solr-n_0000000000) [n:127.0.0.1:54907_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:54907_solr
[junit4] 2> 9392867 WARN (OverseerAutoScalingTriggerThread-98860612339105800-127.0.0.1:54907_solr-n_0000000000) [n:127.0.0.1:54907_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 9392954 INFO (jetty-closer-4996-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@17c31e63{/solr,null,UNAVAILABLE}
[junit4] 2> 9392968 INFO (zkCallback-5018-thread-8-processing-n:127.0.0.1:57520_solr) [n:127.0.0.1:57520_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57520_solr
[junit4] 2> 9392969 INFO (jetty-closer-4996-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7d5539ed{/solr,null,UNAVAILABLE}
[junit4] 2> 9393031 INFO (jetty-closer-4996-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3538438d{/solr,null,UNAVAILABLE}
[junit4] 2> 9393068 ERROR (SUITE-SearchRateTriggerTest-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 9393068 INFO (SUITE-SearchRateTriggerTest-seed#[5D3C760E1FD07E28]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57676 57676
[junit4] 2> 9393158 INFO (Thread-4652) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57676 57676
[junit4] 2> 9393184 WARN (Thread-4652) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 4 /solr/configs/conf
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 53 /solr/collections/SearchRateTriggerTest-collection1/state.json
[junit4] 2> 48 /solr/collections/SearchRateTriggerTest-collection2/state.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 2 /solr/overseer_elect/election/98860612339105800-127.0.0.1:54907_solr-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/live_nodes
[junit4] 2> 6 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.SearchRateTriggerTest_5D3C760E1FD07E28-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1297, maxMBSortInHeap=6.6087307604771155, sim=RandomSimilarity(queryNorm=true): {}, locale=ar-SD, timezone=SystemV/MST7MDT
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=247663976,total=520617984
[junit4] 2> NOTE: All tests run in this JVM: [SuggesterTSTTest, ClusterStateUpdateTest, TestLegacyTerms, HdfsChaosMonkeySafeLeaderTest, NotRequiredUniqueKeyTest, SolrCmdDistributorTest, TestDynamicFieldResource, PrimUtilsTest, InfixSuggestersTest, ImplicitSnitchTest, SynonymTokenizerTest, MoreLikeThisHandlerTest, ExecutePlanActionTest, TestManagedResource, SharedFSAutoReplicaFailoverTest, SOLR749Test, SuggesterFSTTest, TestCollectionAPIs, TestSurroundQueryParser, BasicAuthStandaloneTest, TestReloadAndDeleteDocs, TestFastWriter, TestInitQParser, TestSimpleTrackingShardHandler, SimpleMLTQParserTest, TestDocumentBuilder, TestStressLiveNodes, TestSolrQueryParser, CdcrBootstrapTest, TestInPlaceUpdatesStandalone, NodeMutatorTest, ZkSolrClientTest, ShardRoutingTest, TestBlobHandler, TestDocSet, HdfsRecoverLeaseTest, TestReRankQParserPlugin, QueryEqualityTest, TestDelegationWithHadoopAuth, TestCloudDeleteByQuery, TestSubQueryTransformer, TestMacros, CloneFieldUpdateProcessorFactoryTest, TestFieldCacheSort, UninvertDocValuesMergePolicyTest, TestCollectionsAPIViaSolrCloudCluster, BlobRepositoryCloudTest, TemplateUpdateProcessorTest, TermVectorComponentDistributedTest, HdfsUnloadDistributedZkTest, TestManagedSchema, TestSchemaResource, TestRemoteStreaming, ChaosMonkeySafeLeaderTest, DistributedDebugComponentTest, TestSegmentSorting, DeleteShardTest, TestStressRecovery, SolrMetricManagerTest, ConvertedLegacyTest, OpenCloseCoreStressTest, SolrCloudExampleTest, RequestHandlersTest, TestSolrConfigHandlerConcurrent, TestMissingGroups, TestManagedSynonymFilterFactory, CollectionTooManyReplicasTest, TestStressLucene, TestSolrConfigHandler, AssignTest, TestGeoJSONResponseWriter, LeaderFailureAfterFreshStartTest, HighlighterMaxOffsetTest, TestFastLRUCache, MergeStrategyTest, BooleanFieldTest, DistributedVersionInfoTest, TestJettySolrRunner, ManagedSchemaRoundRobinCloudTest, DirectUpdateHandlerTest, SSLMigrationTest, NumericFieldsTest, TestDistribDocBasedVersion, MoveReplicaHDFSTest, TestRetrieveFieldsOptimizer, SolrGangliaReporterTest, TestDistributedStatsComponentCardinality, CoreAdminRequestStatusTest, CloudExitableDirectoryReaderTest, ChaosMonkeySafeLeaderWithPullReplicasTest, TestCollationField, TestScoreJoinQPScore, TestSolr4Spatial, TestAnalyzedSuggestions, SampleTest, CollectionsAPIDistributedZkTest, TestLazyCores, TestQueryWrapperFilter, NodeAddedTriggerTest, TestCloudSchemaless, TestSkipOverseerOperations, CdcrRequestHandlerTest, SpatialHeatmapFacetsTest, TestAtomicUpdateErrorCases, TestDefaultStatsCache, DistribDocExpirationUpdateProcessorTest, TestHdfsCloudBackupRestore, ConcurrentDeleteAndCreateCollectionTest, TestJmxIntegration, DistributedTermsComponentTest, TestPushWriter, SchemaVersionSpecificBehaviorTest, TestPullReplica, TestMiniSolrCloudClusterSSL, SpatialRPTFieldTypeTest, ExitableDirectoryReaderTest, TestPivotHelperCode, TestConfigSetsAPIZkFailure, OpenExchangeRatesOrgProviderTest, PolyFieldTest, LukeRequestHandlerTest, TestOmitPositions, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, FieldAnalysisRequestHandlerTest, RegexBoostProcessorTest, ReturnFieldsTest, QueryParsingTest, TestPartialUpdateDeduplication, CSVRequestHandlerTest, TestComponentsName, HighlighterConfigTest, TestSolrIndexConfig, CopyFieldTest, ResponseLogComponentTest, BadComponentTest, TestSolrDeletionPolicy2, TestFuzzyAnalyzedSuggestions, DirectSolrConnectionTest, TestCodecSupport, FileUtilsTest, TestRTGBase, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestHighlightDedupGrouping, AliasIntegrationTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, DeleteLastCustomShardedReplicaTest, DeleteReplicaTest, LeaderInitiatedRecoveryOnCommitTest, MissingSegmentRecoveryTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerStatusTest, ReplicationFactorTest, RollingRestartTest, SolrXmlInZkTest, TestCloudPivotFacet, TestCollectionAPI, TestCryptoKeys, TestLeaderInitiatedRecoveryThread, TestPrepRecovery, TestSizeLimitedDistributedMap, TestSolrCloudWithDelegationTokens, TestStressCloudBlindAtomicUpdates, TestTolerantUpdateProcessorRandomCloud, AutoAddReplicasIntegrationTest, SearchRateTriggerTest]
[junit4] Completed [553/744 (7!)] on J1 in 56.50s, 1 test, 1 failure <<< FAILURES!
[...truncated 47921 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 187 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/187/
4 tests failed.
FAILED: org.apache.solr.schema.TestCloudSchemaless.test
Error Message:
Error from server at http://127.0.0.1:33937: At least one of the node(s) specified are not currently active, no action taken.
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:33937: At least one of the node(s) specified are not currently active, no action taken.
at __randomizedtesting.SeedInfo.seed([84F02A0C6AF774C6:CA415D6C40B193E]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:418)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.schema.TestCloudSchemaless
Error Message:
52 threads leaked from SUITE scope at org.apache.solr.schema.TestCloudSchemaless: 1) Thread[id=4116, name=zkCallback-1786-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=4120, name=Scheduler-730779311, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=4097, name=qtp149521024-4097, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=4155, name=OverseerStateUpdate-98853057374322697-127.0.0.1:49517_-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:202) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=4144, name=TEST-TestCloudSchemaless.test-seed#[84F02A0C6AF774C6]-EventThread, state=WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 6) Thread[id=4138, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=4167, name=zkCallback-1786-thread-5, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=4099, name=qtp149521024-4099-acceptor-0@a8fecb-ServerConnector@24584ad1{HTTP/1.1,[http/1.1]}{127.0.0.1:49517}, state=RUNNABLE, group=TGRP-TestCloudSchemaless] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=4109, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=4102, name=qtp149521024-4102, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=4146, name=Thread-330, state=WAITING, group=TGRP-TestCloudSchemaless] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1673) 12) Thread[id=4168, name=ScheduledTrigger-885-thread-3, state=WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=4125, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=4123, name=searcherExecutor-871-thread-1, state=WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=4103, name=qtp149521024-4103, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=4114, name=TEST-TestCloudSchemaless.test-seed#[84F02A0C6AF774C6]-SendThread(127.0.0.1:56731), state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 17) Thread[id=4162, name=zkCallback-1792-thread-2, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=4165, name=zkCallback-1786-thread-3, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=4128, name=qtp139147000-4128-acceptor-0@2765930c-ServerConnector@3b1d046{HTTP/1.1,[http/1.1]}{127.0.0.1:33937}, state=RUNNABLE, group=TGRP-TestCloudSchemaless] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=4143, name=TEST-TestCloudSchemaless.test-seed#[84F02A0C6AF774C6]-SendThread(127.0.0.1:56731), state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 21) Thread[id=4156, name=OverseerCollectionConfigSetProcessor-98853057374322697-127.0.0.1:49517_-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=4100, name=qtp149521024-4100, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=4110, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=4163, name=zkCallback-1792-thread-3, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=4160, name=ScheduledTrigger-885-thread-2, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:389) at org.apache.solr.cloud.ZkDistributedQueue.offer(ZkDistributedQueue.java:293) at org.apache.solr.cloud.autoscaling.TriggerEventQueue.offerEvent(TriggerEventQueue.java:58) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.enqueue(ScheduledTriggers.java:411) at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:205) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$78/321192884.process(Unknown Source) at org.apache.solr.cloud.autoscaling.NodeLostTrigger.run(NodeLostTrigger.java:163) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:453) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=4118, name=coreContainerWorkExecutor-866-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=4169, name=ScheduledTrigger-885-thread-4, state=WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=4145, name=zkCallback-1792-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=4130, name=qtp139147000-4130, state=RUNNABLE, group=TGRP-TestCloudSchemaless] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=4132, name=qtp139147000-4132, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=4134, name=org.eclipse.jetty.server.session.HashSessionManager@b962002Timer, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=4148, name=Scheduler-85887704, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=4117, name=Thread-321, state=WAITING, group=TGRP-TestCloudSchemaless] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1673) 34) Thread[id=4139, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=4159, name=ScheduledTrigger-885-thread-1, state=WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=4105, name=org.eclipse.jetty.server.session.HashSessionManager@67deb94dTimer, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 37) Thread[id=4101, name=qtp149521024-4101, state=RUNNABLE, group=TGRP-TestCloudSchemaless] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 38) Thread[id=4126, name=qtp139147000-4126, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=4154, name=zkCallback-1786-thread-2, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 40) Thread[id=4127, name=qtp139147000-4127, state=RUNNABLE, group=TGRP-TestCloudSchemaless] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 41) Thread[id=4121, name=watches-1787-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 42) Thread[id=4147, name=coreContainerWorkExecutor-877-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 43) Thread[id=4133, name=qtp139147000-4133, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 44) Thread[id=4129, name=qtp139147000-4129, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 45) Thread[id=4157, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 46) Thread[id=4164, name=zkCallback-1792-thread-4, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 47) Thread[id=4098, name=qtp149521024-4098, state=RUNNABLE, group=TGRP-TestCloudSchemaless] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 48) Thread[id=4104, name=qtp149521024-4104, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 49) Thread[id=4131, name=qtp139147000-4131, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 50) Thread[id=4115, name=TEST-TestCloudSchemaless.test-seed#[84F02A0C6AF774C6]-EventThread, state=WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 51) Thread[id=4158, name=OverseerAutoScalingTriggerThread-98853057374322697-127.0.0.1:49517_-n_0000000001, state=WAITING, group=Overseer autoscaling triggers] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:161) at java.lang.Thread.run(Thread.java:748) 52) Thread[id=4166, name=zkCallback-1786-thread-4, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 52 threads leaked from SUITE scope at org.apache.solr.schema.TestCloudSchemaless:
1) Thread[id=4116, name=zkCallback-1786-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=4120, name=Scheduler-730779311, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=4097, name=qtp149521024-4097, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=4155, name=OverseerStateUpdate-98853057374322697-127.0.0.1:49517_-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:202)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=4144, name=TEST-TestCloudSchemaless.test-seed#[84F02A0C6AF774C6]-EventThread, state=WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
6) Thread[id=4138, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=4167, name=zkCallback-1786-thread-5, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=4099, name=qtp149521024-4099-acceptor-0@a8fecb-ServerConnector@24584ad1{HTTP/1.1,[http/1.1]}{127.0.0.1:49517}, state=RUNNABLE, group=TGRP-TestCloudSchemaless]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=4109, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=4102, name=qtp149521024-4102, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=4146, name=Thread-330, state=WAITING, group=TGRP-TestCloudSchemaless]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1673)
12) Thread[id=4168, name=ScheduledTrigger-885-thread-3, state=WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=4125, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=4123, name=searcherExecutor-871-thread-1, state=WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=4103, name=qtp149521024-4103, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
16) Thread[id=4114, name=TEST-TestCloudSchemaless.test-seed#[84F02A0C6AF774C6]-SendThread(127.0.0.1:56731), state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
17) Thread[id=4162, name=zkCallback-1792-thread-2, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
18) Thread[id=4165, name=zkCallback-1786-thread-3, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
19) Thread[id=4128, name=qtp139147000-4128-acceptor-0@2765930c-ServerConnector@3b1d046{HTTP/1.1,[http/1.1]}{127.0.0.1:33937}, state=RUNNABLE, group=TGRP-TestCloudSchemaless]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
20) Thread[id=4143, name=TEST-TestCloudSchemaless.test-seed#[84F02A0C6AF774C6]-SendThread(127.0.0.1:56731), state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
21) Thread[id=4156, name=OverseerCollectionConfigSetProcessor-98853057374322697-127.0.0.1:49517_-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)
at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
at java.lang.Thread.run(Thread.java:748)
22) Thread[id=4100, name=qtp149521024-4100, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
23) Thread[id=4110, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
24) Thread[id=4163, name=zkCallback-1792-thread-3, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
25) Thread[id=4160, name=ScheduledTrigger-885-thread-2, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:389)
at org.apache.solr.cloud.ZkDistributedQueue.offer(ZkDistributedQueue.java:293)
at org.apache.solr.cloud.autoscaling.TriggerEventQueue.offerEvent(TriggerEventQueue.java:58)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.enqueue(ScheduledTriggers.java:411)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:205)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers$$Lambda$78/321192884.process(Unknown Source)
at org.apache.solr.cloud.autoscaling.NodeLostTrigger.run(NodeLostTrigger.java:163)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:453)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
26) Thread[id=4118, name=coreContainerWorkExecutor-866-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
27) Thread[id=4169, name=ScheduledTrigger-885-thread-4, state=WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
28) Thread[id=4145, name=zkCallback-1792-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
29) Thread[id=4130, name=qtp139147000-4130, state=RUNNABLE, group=TGRP-TestCloudSchemaless]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
30) Thread[id=4132, name=qtp139147000-4132, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
31) Thread[id=4134, name=org.eclipse.jetty.server.session.HashSessionManager@b962002Timer, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
32) Thread[id=4148, name=Scheduler-85887704, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
33) Thread[id=4117, name=Thread-321, state=WAITING, group=TGRP-TestCloudSchemaless]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1673)
34) Thread[id=4139, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
35) Thread[id=4159, name=ScheduledTrigger-885-thread-1, state=WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
36) Thread[id=4105, name=org.eclipse.jetty.server.session.HashSessionManager@67deb94dTimer, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
37) Thread[id=4101, name=qtp149521024-4101, state=RUNNABLE, group=TGRP-TestCloudSchemaless]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
38) Thread[id=4126, name=qtp139147000-4126, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
39) Thread[id=4154, name=zkCallback-1786-thread-2, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
40) Thread[id=4127, name=qtp139147000-4127, state=RUNNABLE, group=TGRP-TestCloudSchemaless]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
41) Thread[id=4121, name=watches-1787-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
42) Thread[id=4147, name=coreContainerWorkExecutor-877-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
43) Thread[id=4133, name=qtp139147000-4133, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
44) Thread[id=4129, name=qtp139147000-4129, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
45) Thread[id=4157, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
46) Thread[id=4164, name=zkCallback-1792-thread-4, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
47) Thread[id=4098, name=qtp149521024-4098, state=RUNNABLE, group=TGRP-TestCloudSchemaless]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
48) Thread[id=4104, name=qtp149521024-4104, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
49) Thread[id=4131, name=qtp139147000-4131, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
50) Thread[id=4115, name=TEST-TestCloudSchemaless.test-seed#[84F02A0C6AF774C6]-EventThread, state=WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
51) Thread[id=4158, name=OverseerAutoScalingTriggerThread-98853057374322697-127.0.0.1:49517_-n_0000000001, state=WAITING, group=Overseer autoscaling triggers]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:161)
at java.lang.Thread.run(Thread.java:748)
52) Thread[id=4166, name=zkCallback-1786-thread-4, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([84F02A0C6AF774C6]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.schema.TestCloudSchemaless
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=4145, name=zkCallback-1792-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=4130, name=qtp139147000-4130, state=RUNNABLE, group=TGRP-TestCloudSchemaless] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=4134, name=org.eclipse.jetty.server.session.HashSessionManager@b962002Timer, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=4148, name=Scheduler-85887704, state=WAITING, group=TGRP-TestCloudSchemaless] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=4116, name=zkCallback-1786-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless] at sun.misc
[...truncated too long message...]
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 21) Thread[id=4164, name=zkCallback-1792-thread-4, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 22) Thread[id=4165, name=zkCallback-1786-thread-3, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 23) Thread[id=4098, name=qtp149521024-4098, state=RUNNABLE, group=TGRP-TestCloudSchemaless]
[junit4] > at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 24) Thread[id=4143, name=TEST-TestCloudSchemaless.test-seed#[84F02A0C6AF774C6]-SendThread(127.0.0.1:56731), state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > 25) Thread[id=4163, name=zkCallback-1792-thread-3, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 26) Thread[id=4160, name=ScheduledTrigger-885-thread-2, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 27) Thread[id=4166, name=zkCallback-1786-thread-4, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 28) Thread[id=4118, name=coreContainerWorkExecutor-866-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSchemaless]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 29) Thread[id=4169, name=ScheduledTrigger-885-thread-4, state=WAITING, group=TGRP-TestCloudSchemaless]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([84F02A0C6AF774C6]:0)
[junit4] Completed [74/743 (2!)] on J0 in 54.37s, 1 test, 3 errors <<< FAILURES!
[...truncated 49431 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 186 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/186/
5 tests failed.
FAILED: org.apache.solr.cloud.AliasIntegrationTest.testErrorChecks
Error Message:
Error from server at https://127.0.0.1:45039/solr: deletealias the collection time out:180s
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:45039/solr: deletealias the collection time out:180s
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.AliasIntegrationTest.testErrorChecks(AliasIntegrationTest.java:201)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.core.TestLazyCores.testNoCommit
Error Message:
Exception during query
Stack Trace:
java.lang.RuntimeException: Exception during query
at __randomizedtesting.SeedInfo.seed([7235E95837BA8EE0:AD554889FC9DED45]:0)
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:884)
at org.apache.solr.core.TestLazyCores.check10(TestLazyCores.java:847)
at org.apache.solr.core.TestLazyCores.testNoCommit(TestLazyCores.java:829)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound='10']
xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int><lst name="params"><str name="q">*:*</str></lst></lst><result name="response" numFound="0" start="0"></result>
</response>
request was:q=*:*
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:877)
... 41 more
FAILED: junit.framework.TestSuite.org.apache.solr.core.TestLazyCores
Error Message:
1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 1) Thread[id=10808, name=searcherExecutor-4800-thread-1, state=WAITING, group=TGRP-TestLazyCores] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:
1) Thread[id=10808, name=searcherExecutor-4800-thread-1, state=WAITING, group=TGRP-TestLazyCores]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([7235E95837BA8EE0]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.core.TestLazyCores
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=10808, name=searcherExecutor-4800-thread-1, state=WAITING, group=TGRP-TestLazyCores] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=10808, name=searcherExecutor-4800-thread-1, state=WAITING, group=TGRP-TestLazyCores]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([7235E95837BA8EE0]:0)
FAILED: org.apache.solr.common.cloud.TestZkConfigManager.testUploadConfig
Error Message:
Error checking path /solr
Stack Trace:
java.io.IOException: Error checking path /solr
at __randomizedtesting.SeedInfo.seed([93124FED73827EF:E6C7C56A6BC7122D]:0)
at org.apache.solr.cloud.ZkTestServer.ensurePathExists(ZkTestServer.java:423)
at org.apache.solr.common.cloud.TestZkConfigManager.testUploadConfig(TestZkConfigManager.java:65)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss for /solr
at org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:310)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:512)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:467)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:454)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:410)
at org.apache.solr.cloud.ZkTestServer.ensurePathExists(ZkTestServer.java:421)
... 40 more
Build Log:
[...truncated 12737 lines...]
[junit4] Suite: org.apache.solr.cloud.AliasIntegrationTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_7235E95837BA8EE0-001/init-core-data-001
[junit4] 2> 3197922 WARN (SUITE-AliasIntegrationTest-seed#[7235E95837BA8EE0]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
[junit4] 2> 3197922 INFO (SUITE-AliasIntegrationTest-seed#[7235E95837BA8EE0]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 3197973 INFO (SUITE-AliasIntegrationTest-seed#[7235E95837BA8EE0]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 3197973 INFO (SUITE-AliasIntegrationTest-seed#[7235E95837BA8EE0]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 3197973 INFO (SUITE-AliasIntegrationTest-seed#[7235E95837BA8EE0]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_7235E95837BA8EE0-001/tempDir-001
[junit4] 2> 3197973 INFO (SUITE-AliasIntegrationTest-seed#[7235E95837BA8EE0]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3198010 INFO (Thread-4139) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3198010 INFO (Thread-4139) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3198086 ERROR (Thread-4139) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3198135 INFO (SUITE-AliasIntegrationTest-seed#[7235E95837BA8EE0]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:38990
[junit4] 2> 3198398 INFO (jetty-launcher-3610-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3198432 INFO (jetty-launcher-3610-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3198566 INFO (jetty-launcher-3610-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5730b9a8{/solr,null,AVAILABLE}
[junit4] 2> 3198566 INFO (jetty-launcher-3610-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4156b982{SSL,[ssl, http/1.1]}{127.0.0.1:36898}
[junit4] 2> 3198567 INFO (jetty-launcher-3610-thread-2) [ ] o.e.j.s.Server Started @3213106ms
[junit4] 2> 3198567 INFO (jetty-launcher-3610-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=36898}
[junit4] 2> 3198567 ERROR (jetty-launcher-3610-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3198567 INFO (jetty-launcher-3610-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 3198567 INFO (jetty-launcher-3610-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3198567 INFO (jetty-launcher-3610-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 3198567 INFO (jetty-launcher-3610-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-18T16:00:28.096Z
[junit4] 2> 3198569 INFO (jetty-launcher-3610-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2157c6d{/solr,null,AVAILABLE}
[junit4] 2> 3198569 INFO (jetty-launcher-3610-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@55ad1f{SSL,[ssl, http/1.1]}{127.0.0.1:45039}
[junit4] 2> 3198569 INFO (jetty-launcher-3610-thread-1) [ ] o.e.j.s.Server Started @3213109ms
[junit4] 2> 3198569 INFO (jetty-launcher-3610-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45039}
[junit4] 2> 3198569 ERROR (jetty-launcher-3610-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3198569 INFO (jetty-launcher-3610-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 3198569 INFO (jetty-launcher-3610-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3198569 INFO (jetty-launcher-3610-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 3198570 INFO (jetty-launcher-3610-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-18T16:00:28.099Z
[junit4] 2> 3198641 INFO (jetty-launcher-3610-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3198749 INFO (jetty-launcher-3610-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3198802 INFO (jetty-launcher-3610-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38990/solr
[junit4] 2> 3198823 INFO (jetty-launcher-3610-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38990/solr
[junit4] 2> 3199116 INFO (jetty-launcher-3610-thread-1) [n:127.0.0.1:45039_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3199116 INFO (jetty-launcher-3610-thread-1) [n:127.0.0.1:45039_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45039_solr
[junit4] 2> 3199117 INFO (jetty-launcher-3610-thread-1) [n:127.0.0.1:45039_solr ] o.a.s.c.Overseer Overseer (id=98850729334603782-127.0.0.1:45039_solr-n_0000000000) starting
[junit4] 2> 3199337 INFO (jetty-launcher-3610-thread-1) [n:127.0.0.1:45039_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45039_solr
[junit4] 2> 3199354 INFO (zkCallback-3623-thread-1-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3199932 INFO (jetty-launcher-3610-thread-2) [n:127.0.0.1:36898_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3199933 INFO (jetty-launcher-3610-thread-2) [n:127.0.0.1:36898_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3199935 INFO (jetty-launcher-3610-thread-2) [n:127.0.0.1:36898_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36898_solr
[junit4] 2> 3199948 INFO (zkCallback-3623-thread-1-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3200087 INFO (zkCallback-3620-thread-1-processing-n:127.0.0.1:36898_solr) [n:127.0.0.1:36898_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3200835 INFO (jetty-launcher-3610-thread-2) [n:127.0.0.1:36898_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36898.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@50cd0791
[junit4] 2> 3201002 INFO (jetty-launcher-3610-thread-2) [n:127.0.0.1:36898_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36898.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@50cd0791
[junit4] 2> 3201078 INFO (jetty-launcher-3610-thread-1) [n:127.0.0.1:45039_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45039.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@50cd0791
[junit4] 2> 3201092 INFO (jetty-launcher-3610-thread-2) [n:127.0.0.1:36898_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36898.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@50cd0791
[junit4] 2> 3201093 INFO (jetty-launcher-3610-thread-2) [n:127.0.0.1:36898_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_7235E95837BA8EE0-001/tempDir-001/node2/.
[junit4] 2> 3201302 WARN (jetty-launcher-3610-thread-1) [n:127.0.0.1:45039_solr ] o.a.s.m.r.j.JmxMetricsReporter Unable to register gauge
[junit4] 2> javax.management.InstanceNotFoundException: solr_45039:dom1=solr,dom2=jvm,name0=memory,name1=pools,name2=PS-Survivor-Space,name=used-after-gc
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1095)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:427)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
[junit4] 2> at org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.registerMBean(JmxMetricsReporter.java:530)
[junit4] 2> at org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.onGaugeAdded(JmxMetricsReporter.java:565)
[junit4] 2> at org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.lambda$start$0(JmxMetricsReporter.java:738)
[junit4] 2> at java.util.HashMap.forEach(HashMap.java:1288)
[junit4] 2> at org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.start(JmxMetricsReporter.java:728)
[junit4] 2> at org.apache.solr.metrics.reporters.SolrJmxReporter.doInit(SolrJmxReporter.java:109)
[junit4] 2> at org.apache.solr.metrics.SolrMetricReporter.init(SolrMetricReporter.java:70)
[junit4] 2> at org.apache.solr.metrics.SolrMetricManager.loadReporter(SolrMetricManager.java:849)
[junit4] 2> at org.apache.solr.metrics.SolrMetricManager.loadReporters(SolrMetricManager.java:817)
[junit4] 2> at org.apache.solr.core.CoreContainer.load(CoreContainer.java:549)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)
[junit4] 2> at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)
[junit4] 2> at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
[junit4] 2> at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
[junit4] 2> at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
[junit4] 2> at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)
[junit4] 2> at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3201305 INFO (jetty-launcher-3610-thread-1) [n:127.0.0.1:45039_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45039.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@50cd0791
[junit4] 2> 3201305 INFO (jetty-launcher-3610-thread-1) [n:127.0.0.1:45039_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45039.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@50cd0791
[junit4] 2> 3201376 INFO (jetty-launcher-3610-thread-1) [n:127.0.0.1:45039_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_7235E95837BA8EE0-001/tempDir-001/node1/.
[junit4] 2> 3201770 INFO (SUITE-AliasIntegrationTest-seed#[7235E95837BA8EE0]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3201859 INFO (SUITE-AliasIntegrationTest-seed#[7235E95837BA8EE0]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38990/solr ready
[junit4] 2> 3201956 INFO (TEST-AliasIntegrationTest.test-seed#[7235E95837BA8EE0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 3202178 INFO (qtp1112567442-16117) [n:127.0.0.1:45039_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3202226 INFO (OverseerThreadFactory-4574-thread-1-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 3202408 INFO (OverseerStateUpdate-98850729334603782-127.0.0.1:45039_solr-n_0000000000) [n:127.0.0.1:45039_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"collection1_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:45039/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 3202482 INFO (OverseerStateUpdate-98850729334603782-127.0.0.1:45039_solr-n_0000000000) [n:127.0.0.1:45039_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"collection1_shard2_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:36898/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 3203094 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3203094 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 3203392 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=collection1_shard2_replica_n2&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3203392 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 3203421 INFO (zkCallback-3620-thread-1-processing-n:127.0.0.1:36898_solr) [n:127.0.0.1:36898_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3203433 INFO (zkCallback-3623-thread-1-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3203433 INFO (zkCallback-3623-thread-2-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3203584 INFO (zkCallback-3620-thread-1-processing-n:127.0.0.1:36898_solr) [n:127.0.0.1:36898_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3203596 INFO (zkCallback-3623-thread-1-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3203596 INFO (zkCallback-3623-thread-2-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3203633 INFO (zkCallback-3620-thread-2-processing-n:127.0.0.1:36898_solr) [n:127.0.0.1:36898_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3204570 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 3204660 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 3204694 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.IndexSchema [collection1_shard2_replica_n2] Schema name=minimal
[junit4] 2> 3204696 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3204696 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n2' using configuration from collection collection1, trusted=true
[junit4] 2> 3204697 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36898.solr.core.collection1.shard2.replica_n2' (registry 'solr.core.collection1.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@50cd0791
[junit4] 2> 3204697 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3204697 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrCore [[collection1_shard2_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_7235E95837BA8EE0-001/tempDir-001/node2/collection1_shard2_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_7235E95837BA8EE0-001/tempDir-001/node2/./collection1_shard2_replica_n2/data/]
[junit4] 2> 3204806 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema [collection1_shard1_replica_n1] Schema name=minimal
[junit4] 2> 3204987 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3204988 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n1' using configuration from collection collection1, trusted=true
[junit4] 2> 3204988 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45039.solr.core.collection1.shard1.replica_n1' (registry 'solr.core.collection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@50cd0791
[junit4] 2> 3204988 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3204988 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [[collection1_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_7235E95837BA8EE0-001/tempDir-001/node1/collection1_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_7235E95837BA8EE0-001/tempDir-001/node1/./collection1_shard1_replica_n1/data/]
[junit4] 2> 3205686 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3205686 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3205687 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3205687 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3205702 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@5e1d0950[collection1_shard2_replica_n2] main]
[junit4] 2> 3205705 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 3205705 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 3205706 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3205792 INFO (searcherExecutor-4579-thread-1-processing-n:127.0.0.1:36898_solr x:collection1_shard2_replica_n2 s:shard2 c:collection1 r:core_node4) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrCore [collection1_shard2_replica_n2] Registered new searcher Searcher@5e1d0950[collection1_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3205792 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581611677459152896
[junit4] 2> 3205813 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3205813 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3205813 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:36898/solr/collection1_shard2_replica_n2/
[junit4] 2> 3205813 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3205813 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SyncStrategy https://127.0.0.1:36898/solr/collection1_shard2_replica_n2/ has no replicas
[junit4] 2> 3205813 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3205829 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:36898/solr/collection1_shard2_replica_n2/ shard2
[junit4] 2> 3205831 INFO (zkCallback-3620-thread-1-processing-n:127.0.0.1:36898_solr) [n:127.0.0.1:36898_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3205831 INFO (zkCallback-3623-thread-3-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3205831 INFO (zkCallback-3623-thread-2-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3205832 INFO (zkCallback-3620-thread-2-processing-n:127.0.0.1:36898_solr) [n:127.0.0.1:36898_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3205884 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3205994 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3205995 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3205995 INFO (qtp1994399353-16125) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=collection1_shard2_replica_n2&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2603
[junit4] 2> 3205996 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3205996 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3206011 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1c512120[collection1_shard1_replica_n1] main]
[junit4] 2> 3206011 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 3206012 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 3206012 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3206013 INFO (searcherExecutor-4580-thread-1-processing-n:127.0.0.1:45039_solr x:collection1_shard1_replica_n1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [collection1_shard1_replica_n1] Registered new searcher Searcher@1c512120[collection1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3206013 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581611677690888192
[junit4] 2> 3206067 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3206067 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3206067 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:45039/solr/collection1_shard1_replica_n1/
[junit4] 2> 3206067 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3206067 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:45039/solr/collection1_shard1_replica_n1/ has no replicas
[junit4] 2> 3206067 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3206121 INFO (zkCallback-3620-thread-2-processing-n:127.0.0.1:36898_solr) [n:127.0.0.1:36898_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3206121 INFO (zkCallback-3620-thread-1-processing-n:127.0.0.1:36898_solr) [n:127.0.0.1:36898_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3206121 INFO (zkCallback-3623-thread-2-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3206121 INFO (zkCallback-3623-thread-3-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3206123 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:45039/solr/collection1_shard1_replica_n1/ shard1
[junit4] 2> 3206226 INFO (zkCallback-3620-thread-2-processing-n:127.0.0.1:36898_solr) [n:127.0.0.1:36898_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3206226 INFO (zkCallback-3620-thread-1-processing-n:127.0.0.1:36898_solr) [n:127.0.0.1:36898_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3206227 INFO (zkCallback-3623-thread-3-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3206227 INFO (zkCallback-3623-thread-2-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3206283 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3206284 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3190
[junit4] 2> 3206393 INFO (zkCallback-3620-thread-2-processing-n:127.0.0.1:36898_solr) [n:127.0.0.1:36898_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3206393 INFO (zkCallback-3620-thread-1-processing-n:127.0.0.1:36898_solr) [n:127.0.0.1:36898_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3206394 INFO (zkCallback-3623-thread-3-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3206394 INFO (zkCallback-3623-thread-2-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3206409 INFO (qtp1112567442-16117) [n:127.0.0.1:45039_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3206409 INFO (qtp1112567442-16117) [n:127.0.0.1:45039_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=4231
[junit4] 2> 3206410 INFO (qtp1112567442-16118) [n:127.0.0.1:45039_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=collection2&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3206616 INFO (OverseerThreadFactory-4574-thread-2-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.CreateCollectionCmd Create collection collection2
[junit4] 2> 3206653 INFO (OverseerCollectionConfigSetProcessor-98850729334603782-127.0.0.1:45039_solr-n_0000000000) [n:127.0.0.1:45039_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3207141 INFO (OverseerStateUpdate-98850729334603782-127.0.0.1:45039_solr-n_0000000000) [n:127.0.0.1:45039_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"collection2",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"collection2_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:45039/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 3207300 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=collection2_shard1_replica_n1&action=CREATE&numShards=1&collection=collection2&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3207454 INFO (zkCallback-3623-thread-3-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection2/state.json] for collection [collection2] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3207454 INFO (zkCallback-3623-thread-2-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection2/state.json] for collection [collection2] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3208440 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 3208573 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.s.IndexSchema [collection2_shard1_replica_n1] Schema name=minimal
[junit4] 2> 3208588 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3208588 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection2_shard1_replica_n1' using configuration from collection collection2, trusted=true
[junit4] 2> 3208588 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45039.solr.core.collection2.shard1.replica_n1' (registry 'solr.core.collection2.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@50cd0791
[junit4] 2> 3208589 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3208589 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.SolrCore [[collection2_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_7235E95837BA8EE0-001/tempDir-001/node1/collection2_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AliasIntegrationTest_7235E95837BA8EE0-001/tempDir-001/node1/./collection2_shard1_replica_n1/data/]
[junit4] 2> 3209303 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3209303 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3209305 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3209305 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3209319 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@718b125[collection2_shard1_replica_n1] main]
[junit4] 2> 3209340 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 3209340 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 3209341 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3209341 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581611681180549120
[junit4] 2> 3209357 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3209357 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3209357 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:45039/solr/collection2_shard1_replica_n1/
[junit4] 2> 3209357 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3209357 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:45039/solr/collection2_shard1_replica_n1/ has no replicas
[junit4] 2> 3209357 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3209358 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:45039/solr/collection2_shard1_replica_n1/ shard1
[junit4] 2> 3209359 INFO (searcherExecutor-4589-thread-1-processing-n:127.0.0.1:45039_solr x:collection2_shard1_replica_n1 s:shard1 c:collection2 r:core_node2) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.SolrCore [collection2_shard1_replica_n1] Registered new searcher Searcher@718b125[collection2_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3209483 INFO (zkCallback-3623-thread-3-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection2/state.json] for collection [collection2] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3209483 INFO (zkCallback-3623-thread-2-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection2/state.json] for collection [collection2] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3209522 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3209575 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=collection2_shard1_replica_n1&action=CREATE&numShards=1&collection=collection2&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2275
[junit4] 2> 3209644 INFO (qtp1112567442-16118) [n:127.0.0.1:45039_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3209681 INFO (zkCallback-3623-thread-2-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection2/state.json] for collection [collection2] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3209681 INFO (zkCallback-3623-thread-3-processing-n:127.0.0.1:45039_solr) [n:127.0.0.1:45039_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection2/state.json] for collection [collection2] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3210661 INFO (qtp1112567442-16118) [n:127.0.0.1:45039_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=collection2&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=4250
[junit4] 2> 3210788 INFO (qtp1112567442-16116) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=collection1:7&wt=javabin&version=2}{add=[8 (1581611682642264064)]} 0 52
[junit4] 2> 3210917 INFO (OverseerCollectionConfigSetProcessor-98850729334603782-127.0.0.1:45039_solr-n_0000000000) [n:127.0.0.1:45039_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3211245 INFO (qtp1994399353-16127) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n2] webapp=/solr path=/update params={_stateVer_=collection1:7&wt=javabin&version=2}{add=[6 (1581611683174940672), 7 (1581611683177037824)]} 0 1
[junit4] 2> 3211297 INFO (qtp1112567442-16119) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581611683231563776,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 3211421 INFO (qtp1112567442-16119) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@29130d70 commitCommandVersion:1581611683231563776
[junit4] 2> 3211480 INFO (qtp1112567442-16119) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@bf6c7b8[collection1_shard1_replica_n1] main]
[junit4] 2> 3211481 INFO (qtp1112567442-16119) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 3211494 INFO (qtp1994399353-16120) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581611683438133248,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 3211497 INFO (qtp1994399353-16120) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@359f5c9c commitCommandVersion:1581611683438133248
[junit4] 2> 3211537 INFO (searcherExecutor-4580-thread-1-processing-n:127.0.0.1:45039_solr x:collection1_shard1_replica_n1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [collection1_shard1_replica_n1] Registered new searcher Searcher@bf6c7b8[collection1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C1)))}
[junit4] 2> 3211537 INFO (qtp1112567442-16119) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:45039/solr/collection1_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 240
[junit4] 2> 3211590 INFO (qtp1994399353-16120) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@5164e5d[collection1_shard2_replica_n2] main]
[junit4] 2> 3211591 INFO (qtp1994399353-16120) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 3211625 INFO (searcherExecutor-4579-thread-1-processing-n:127.0.0.1:36898_solr x:collection1_shard2_replica_n2 s:shard2 c:collection1 r:core_node4) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrCore [collection1_shard2_replica_n2] Registered new searcher Searcher@5164e5d[collection1_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C2)))}
[junit4] 2> 3211625 INFO (qtp1994399353-16120) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:45039/solr/collection1_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 131
[junit4] 2> 3211625 INFO (qtp1112567442-16113) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=collection1:7&commit=true&wt=javabin&version=2}{commit=} 0 347
[junit4] 2> 3211647 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection2_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=collection2:4&wt=javabin&version=2}{add=[9 (1581611683576545280), 10 (1581611683578642432)]} 0 21
[junit4] 2> 3211648 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1581611683599613952,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 3211648 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@50c8fce6 commitCommandVersion:1581611683599613952
[junit4] 2> 3211682 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@56bbfebb[collection2_shard1_replica_n1] main]
[junit4] 2> 3211682 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 3211698 INFO (searcherExecutor-4589-thread-1-processing-n:127.0.0.1:45039_solr x:collection2_shard1_replica_n1 s:shard1 c:collection2 r:core_node2) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.SolrCore [collection2_shard1_replica_n1] Registered new searcher Searcher@56bbfebb[collection2_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C2)))}
[junit4] 2> 3211698 INFO (qtp1112567442-16112) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection2_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=collection2:4&commit=true&wt=javabin&version=2}{commit=} 0 50
[junit4] 2> 3211699 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :createalias with params collections=collection1&name=testalias&action=CREATEALIAS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3211844 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collections=collection1&name=testalias&action=CREATEALIAS&wt=javabin&version=2} status=0 QTime=145
[junit4] 2> 3211846 INFO (qtp1112567442-16118) [n:127.0.0.1:45039_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :listaliases with params action=LISTALIASES&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3211846 INFO (qtp1112567442-16118) [n:127.0.0.1:45039_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LISTALIASES&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3211899 INFO (qtp1112567442-16117) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=collection1:7&fl=id&fl=score&shards.purpose=4&start=0&fsv=true&collection=collection1&shard.url=https://127.0.0.1:45039/solr/collection1_shard1_replica_n1/&rows=10&version=2&q=*:*&NOW=1508342441376&isShard=true&wt=javabin} hits=1 status=0 QTime=0
[junit4] 2> 3211969 INFO (qtp1994399353-16126) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=collection1:7&fl=id&fl=score&shards.purpose=4&start=0&fsv=true&collection=collection1&shard.url=https://127.0.0.1:36898/solr/collection1_shard2_replica_n2/&rows=10&version=2&q=*:*&NOW=1508342441376&isShard=true&wt=javabin} hits=2 status=0 QTime=69
[junit4] 2> 3211971 INFO (qtp1112567442-16117) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&_stateVer_=collection1:7&shards.purpose=64&NOW=1508342441376&ids=8&isShard=true&collection=collection1&shard.url=https://127.0.0.1:45039/solr/collection1_shard1_replica_n1/&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3211972 INFO (qtp1994399353-16126) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&_stateVer_=collection1:7&shards.purpose=64&NOW=1508342441376&ids=6,7&isShard=true&collection=collection1&shard.url=https://127.0.0.1:36898/solr/collection1_shard2_replica_n2/&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3211972 INFO (qtp1112567442-16118) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&_stateVer_=collection1:7&wt=javabin&version=2} hits=3 status=0 QTime=125
[junit4] 2> 3212045 INFO (qtp1112567442-16116) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1] webapp=/solr path=/select params={df=text&distrib=false&fl=id&fl=score&shards.purpose=4&start=0&fsv=true&collection=collection1&shard.url=https://127.0.0.1:45039/solr/collection1_shard1_replica_n1/&rows=10&version=2&q=*:*&NOW=1508342441574&isShard=true&wt=javabin} hits=1 status=0 QTime=0
[junit4] 2> 3212050 INFO (qtp1994399353-16124) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2] webapp=/solr path=/select params={df=text&distrib=false&fl=id&fl=score&shards.purpose=4&start=0&fsv=true&collection=collection1&shard.url=https://127.0.0.1:36898/solr/collection1_shard2_replica_n2/&rows=10&version=2&q=*:*&NOW=1508342441574&isShard=true&wt=javabin} hits=2 status=0 QTime=0
[junit4] 2> 3212051 INFO (qtp1994399353-16124) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&shards.purpose=64&NOW=1508342441574&ids=6,7&isShard=true&collection=collection1&shard.url=https://127.0.0.1:36898/solr/collection1_shard2_replica_n2/&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3212052 INFO (qtp1112567442-16116) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&shards.purpose=64&NOW=1508342441574&ids=8&isShard=true&collection=collection1&shard.url=https://127.0.0.1:45039/solr/collection1_shard1_replica_n1/&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3212052 INFO (qtp1112567442-16117) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=3 status=0 QTime=7
[junit4] 2> 3212053 INFO (qtp1112567442-16119) [n:127.0.0.1:45039_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :createalias with params collections=collection2,collection1&name=testalias&action=CREATEALIAS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3212104 INFO (OverseerCollectionConfigSetProcessor-98850729334603782-127.0.0.1:45039_solr-n_0000000000) [n:127.0.0.1:45039_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3212323 INFO (qtp1112567442-16119) [n:127.0.0.1:45039_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collections=collection2,collection1&name=testalias&action=CREATEALIAS&wt=javabin&version=2} status=0 QTime=270
[junit4] 2> 3212396 INFO (TEST-AliasIntegrationTest.test-seed#[7235E95837BA8EE0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3212397 INFO (TEST-AliasIntegrationTest.test-seed#[7235E95837BA8EE0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38990/solr ready
[junit4] 2> 3212417 INFO (qtp1994399353-16124) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=collection1:7|collection2:4&fl=id&fl=score&shards.purpose=4&start=0&fsv=true&collection=collection2,collection1&shard.url=https://127.0.0.1:36898/solr/collection1_shard2_replica_n2/&rows=10&version=2&q=*:*&NOW=1508342441938&isShard=true&wt=javabin} hits=2 status=0 QTime=0
[junit4] 2> 3212434 INFO (qtp1112567442-16118) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=collection1:7|collection2:4&fl=id&fl=score&shards.purpose=4&start=0&fsv=true&collection=collection2,collection1&shard.url=https://127.0.0.1:45039/solr/collection1_shard1_replica_n1/&rows=10&version=2&q=*:*&NOW=1508342441938&isShard=true&wt=javabin} hits=1 status=0 QTime=0
[junit4] 2> 3212649 INFO (qtp1112567442-16118) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.S.Request [collection2_shard1_replica_n1] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=collection1:7|collection2:4&fl=id&fl=score&shards.purpose=4&start=0&fsv=true&collection=collection2,collection1&shard.url=https://127.0.0.1:45039/solr/collection2_shard1_replica_n1/&rows=10&version=2&q=*:*&NOW=1508342441938&isShard=true&wt=javabin} hits=2 status=0 QTime=0
[junit4] 2> 3212651 INFO (qtp1994399353-16127) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&_stateVer_=collection1:7|collection2:4&shards.purpose=64&NOW=1508342441938&ids=6,7&isShard=true&collection=collection2,collection1&shard.url=https://127.0.0.1:36898/solr/collection1_shard2_replica_n2/&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3212651 INFO (qtp1112567442-16118) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&_stateVer_=collection1:7|collection2:4&shards.purpose=64&NOW=1508342441938&ids=8&isShard=true&collection=collection2,collection1&shard.url=https://127.0.0.1:45039/solr/collection1_shard1_replica_n1/&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3212686 INFO (qtp1112567442-16118) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.S.Request [collection2_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&_stateVer_=collection1:7|collection2:4&shards.purpose=64&NOW=1508342441938&ids=9,10&isShard=true&collection=collection2,collection1&shard.url=https://127.0.0.1:45039/solr/collection2_shard1_replica_n1/&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3212686 INFO (qtp1994399353-16123) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2] webapp=/solr path=/select params={q=*:*&_stateVer_=collection1:7|collection2:4&collection=testalias&wt=javabin&version=2} hits=5 status=0 QTime=277
[junit4] 2> 3212722 INFO (qtp1112567442-16117) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.S.Request [collection2_shard1_replica_n1] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=collection1:7|collection2:4&fl=id&fl=score&shards.purpose=4&start=0&fsv=true&collection=collection2,collection1&shard.url=https://127.0.0.1:45039/solr/collection2_shard1_replica_n1/&rows=10&version=2&q=*:*&NOW=1508342442230&isShard=true&wt=javabin} hits=2 status=0 QTime=0
[junit4] 2> 3212723 INFO (qtp1994399353-16126) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=collection1:7|collection2:4&fl=id&fl=score&shards.purpose=4&start=0&fsv=true&collection=collection2,collection1&shard.url=https://127.0.0.1:36898/solr/collection1_shard2_replica_n2/&rows=10&version=2&q=*:*&NOW=1508342442230&isShard=true&wt=javabin} hits=2 status=0 QTime=0
[junit4] 2> 3212723 INFO (qtp1112567442-16117) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=collection1:7|collection2:4&fl=id&fl=score&shards.purpose=4&start=0&fsv=true&collection=collection2,collection1&shard.url=https://127.0.0.1:45039/solr/collection1_shard1_replica_n1/&rows=10&version=2&q=*:*&NOW=1508342442230&isShard=true&wt=javabin} hits=1 status=0 QTime=0
[junit4] 2> 3212724 INFO (qtp1994399353-16126) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&_stateVer_=collection1:7|collection2:4&shards.purpose=64&NOW=1508342442230&ids=6,7&isShard=true&collection=collection2,collection1&shard.url=https://127.0.0.1:36898/solr/collection1_shard2_replica_n2/&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3212725 INFO (qtp1112567442-16117) [n:127.0.0.1:45039_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&_stateVer_=collection1:7|collection2:4&shards.purpose=64&NOW=1508342442230&ids=8&isShard=true&collection=collection2,collection1&shard.url=https://127.0.0.1:45039/solr/collection1_shard1_replica_n1/&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3212758 INFO (qtp1112567442-16119) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.S.Request [collection2_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&_stateVer_=collection1:7|collection2:4&shards.purpose=64&NOW=1508342442230&ids=9,10&isShard=true&collection=collection2,collection1&shard.url=https://127.0.0.1:45039/solr/collection2_shard1_replica_n1/&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3212774 INFO (qtp1112567442-16115) [n:127.0.0.1:45039_solr c:collection2 s:shard1 r:core_node2 x:collection2_shard1_replica_n1] o.a.s.c.S.Request [collection2_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&_stateVer_=collection1:7|collection2:4&wt=javabin&version=2} hits=5 status=0 QTime=73
[junit4] 2> 3212996 INFO (qtp1994399353-16126) [n:127.0.0.1:36898_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2] webapp=/solr path=/select params={df=text&distrib=false&fl=id&fl=score&shards.purpose=4&start=0&fsv=true&collection=collection2,collection1&shard.url=https://127.0.0.1:36898/solr/collection1_shard2_replica_n2/&rows=10&version=2
[...truncated too long message...]
unit4] > 1) Thread[id=10808, name=searcherExecutor-4800-thread-1, state=WAITING, group=TGRP-TestLazyCores]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([7235E95837BA8EE0]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=10808, name=searcherExecutor-4800-thread-1, state=WAITING, group=TGRP-TestLazyCores]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([7235E95837BA8EE0]:0)
[junit4] Completed [409/742 (2!)] on J2 in 52.46s, 9 tests, 3 errors <<< FAILURES!
[...truncated 1615 lines...]
[junit4] Suite: org.apache.solr.common.cloud.TestZkConfigManager
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.TestZkConfigManager_93124FED73827EF-001/init-core-data-001
[junit4] 2> 106212 INFO (SUITE-TestZkConfigManager-seed#[93124FED73827EF]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 106213 INFO (SUITE-TestZkConfigManager-seed#[93124FED73827EF]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 106302 INFO (SUITE-TestZkConfigManager-seed#[93124FED73827EF]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 106360 INFO (SUITE-TestZkConfigManager-seed#[93124FED73827EF]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 106374 INFO (Thread-32) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 106431 INFO (Thread-32) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 106702 INFO (SUITE-TestZkConfigManager-seed#[93124FED73827EF]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:44423
[junit4] 2> 106704 INFO (TEST-TestZkConfigManager.testUploadWithACL-seed#[93124FED73827EF]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testUploadWithACL
[junit4] 2> 107630 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
[junit4] 2> 107668 ERROR (Thread-32) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 110184 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f304e586b0000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 111038 INFO (TEST-TestZkConfigManager.testUploadWithACL-seed#[93124FED73827EF]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testUploadWithACL
[junit4] 2> 111041 INFO (TEST-TestZkConfigManager.testConstants-seed#[93124FED73827EF]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testConstants
[junit4] 2> 111041 INFO (TEST-TestZkConfigManager.testConstants-seed#[93124FED73827EF]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testConstants
[junit4] 2> 111055 INFO (TEST-TestZkConfigManager.testUploadConfig-seed#[93124FED73827EF]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testUploadConfig
[junit4] 2> 121953 WARN (TEST-TestZkConfigManager.testUploadConfig-seed#[93124FED73827EF]-SendThread(127.0.0.1:44423)) [ ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 10787ms for sessionid 0x15f304e586b0004
[junit4] 2> 121954 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f304e586b0004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 122065 WARN (zkCallback-31-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@a15cd8b name: ZooKeeperConnection Watcher:127.0.0.1:44423 got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 122065 WARN (zkCallback-31-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 122259 INFO (TEST-TestZkConfigManager.testUploadConfig-seed#[93124FED73827EF]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testUploadConfig
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestZkConfigManager -Dtests.method=testUploadConfig -Dtests.seed=93124FED73827EF -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=hi-IN -Dtests.timezone=Etc/GMT-10 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 11.3s J2 | TestZkConfigManager.testUploadConfig <<<
[junit4] > Throwable #1: java.io.IOException: Error checking path /solr
[junit4] > at __randomizedtesting.SeedInfo.seed([93124FED73827EF:E6C7C56A6BC7122D]:0)
[junit4] > at org.apache.solr.cloud.ZkTestServer.ensurePathExists(ZkTestServer.java:423)
[junit4] > at org.apache.solr.common.cloud.TestZkConfigManager.testUploadConfig(TestZkConfigManager.java:65)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > Caused by: org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss for /solr
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] > at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:310)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:512)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:467)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:454)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:410)
[junit4] > at org.apache.solr.cloud.ZkTestServer.ensurePathExists(ZkTestServer.java:421)
[junit4] > ... 40 more
[junit4] 2> 122313 ERROR (SUITE-TestZkConfigManager-seed#[93124FED73827EF]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 122314 INFO (SUITE-TestZkConfigManager-seed#[93124FED73827EF]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44423 44423
[junit4] 2> 122433 INFO (Thread-32) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44423 44423
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.TestZkConfigManager_93124FED73827EF-001
[junit4] 2> Oct 18, 2017 4:26:17 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=false): {}, locale=hi-IN, timezone=Etc/GMT-10
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=287239344,total=397934592
[junit4] 2> NOTE: All tests run in this JVM: [TestHash, JettyWebappTest, SolrSchemalessExampleTest, SolrParamTest, HyperbolicTangentEvaluatorTest, NotEvaluatorTest, TestToleratedUpdateError, AscEvaluatorTest, TestCoreAdmin, LessThanEvaluatorTest, TestPathTrie, TestZkConfigManager]
[junit4] Completed [28/145 (1!)] on J2 in 17.02s, 3 tests, 1 error <<< FAILURES!
[...truncated 46663 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 185 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/185/
5 tests failed.
FAILED: org.apache.solr.cloud.TestTlogReplica.testCreateDelete {seed=[D68D6E87E44560AB:CD9D9FCB12B5989B]}
Error Message:
KeeperErrorCode = Session expired for /
Stack Trace:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
at org.apache.solr.common.cloud.SolrZkClient.printLayout(SolrZkClient.java:606)
at org.apache.solr.common.cloud.SolrZkClient.printLayoutToStream(SolrZkClient.java:651)
at org.apache.solr.common.cloud.SolrZkClient.printLayoutToStdOut(SolrZkClient.java:646)
at org.apache.solr.cloud.TestTlogReplica.testCreateDelete(TestTlogReplica.java:217)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestTlogReplica.testCreateDelete {seed=[D68D6E87E44560AB:79CB2337267753B7]}
Error Message:
Expected exception SolrException but no exception was thrown
Stack Trace:
junit.framework.AssertionFailedError: Expected exception SolrException but no exception was thrown
at __randomizedtesting.SeedInfo.seed([D68D6E87E44560AB:79CB2337267753B7]:0)
at org.apache.lucene.util.LuceneTestCase.expectThrows(LuceneTestCase.java:2672)
at org.apache.solr.cloud.TestTlogReplica.setUp(TestTlogReplica.java:109)
at sun.reflect.GeneratedMethodAccessor1.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory
Error Message:
replica never fully recovered
Stack Trace:
java.lang.AssertionError: replica never fully recovered
at __randomizedtesting.SeedInfo.seed([D68D6E87E44560AB:BB71CA7A5E0D9FAC]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.waitForRecovery(AutoscalingHistoryHandlerTest.java:303)
at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:255)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication
Error Message:
Index: 0, Size: 0
Stack Trace:
java.lang.IndexOutOfBoundsException: Index: 0, Size: 0
at __randomizedtesting.SeedInfo.seed([D68D6E87E44560AB:C2C535D2C742DDB5]:0)
at java.util.ArrayList.rangeCheck(ArrayList.java:653)
at java.util.ArrayList.get(ArrayList.java:429)
at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication(TestReplicationHandler.java:561)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.common.cloud.TestCollectionStateWatchers.testWaitForStateWatcherIsRetainedOnPredicateFailure
Error Message:
Did not see a fully active cluster after 30 seconds
Stack Trace:
java.lang.AssertionError: Did not see a fully active cluster after 30 seconds
at __randomizedtesting.SeedInfo.seed([DA5E219E64092448:526883CDBCA6CC5A]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.common.cloud.TestCollectionStateWatchers.testWaitForStateWatcherIsRetainedOnPredicateFailure(TestCollectionStateWatchers.java:250)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11847 lines...]
[junit4] Suite: org.apache.solr.cloud.TestTlogReplica
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_D68D6E87E44560AB-001/init-core-data-001
[junit4] 2> 579032 WARN (SUITE-TestTlogReplica-seed#[D68D6E87E44560AB]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
[junit4] 2> 579032 INFO (SUITE-TestTlogReplica-seed#[D68D6E87E44560AB]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 579033 INFO (SUITE-TestTlogReplica-seed#[D68D6E87E44560AB]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 579067 INFO (SUITE-TestTlogReplica-seed#[D68D6E87E44560AB]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 579067 INFO (SUITE-TestTlogReplica-seed#[D68D6E87E44560AB]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_D68D6E87E44560AB-001/tempDir-001
[junit4] 2> 579067 INFO (SUITE-TestTlogReplica-seed#[D68D6E87E44560AB]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 579068 INFO (Thread-463) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 579068 INFO (Thread-463) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 579120 ERROR (Thread-463) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 579177 INFO (SUITE-TestTlogReplica-seed#[D68D6E87E44560AB]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:34377
[junit4] 2> 579304 INFO (jetty-launcher-615-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 579361 INFO (jetty-launcher-615-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 579449 INFO (jetty-launcher-615-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@74db45c0{/solr,null,AVAILABLE}
[junit4] 2> 579451 INFO (jetty-launcher-615-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@20e7dc06{SSL,[ssl, http/1.1]}{127.0.0.1:38739}
[junit4] 2> 579451 INFO (jetty-launcher-615-thread-1) [ ] o.e.j.s.Server Started @596258ms
[junit4] 2> 579451 INFO (jetty-launcher-615-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38739}
[junit4] 2> 579451 ERROR (jetty-launcher-615-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 579451 INFO (jetty-launcher-615-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 579452 INFO (jetty-launcher-615-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 579452 INFO (jetty-launcher-615-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 579452 INFO (jetty-launcher-615-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-18T06:38:15.003Z
[junit4] 2> 579653 INFO (jetty-launcher-615-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@66d53a3{/solr,null,AVAILABLE}
[junit4] 2> 579653 INFO (jetty-launcher-615-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4729e51a{SSL,[ssl, http/1.1]}{127.0.0.1:37548}
[junit4] 2> 579653 INFO (jetty-launcher-615-thread-2) [ ] o.e.j.s.Server Started @596460ms
[junit4] 2> 579653 INFO (jetty-launcher-615-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37548}
[junit4] 2> 579653 ERROR (jetty-launcher-615-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 579654 INFO (jetty-launcher-615-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 579654 INFO (jetty-launcher-615-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 579654 INFO (jetty-launcher-615-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 579654 INFO (jetty-launcher-615-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-18T06:38:15.205Z
[junit4] 2> 579779 INFO (jetty-launcher-615-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 579853 INFO (jetty-launcher-615-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34377/solr
[junit4] 2> 579877 INFO (jetty-launcher-615-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 579946 INFO (jetty-launcher-615-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34377/solr
[junit4] 2> 580021 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f2e34366a0003, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 580203 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f2e34366a0004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 580424 INFO (jetty-launcher-615-thread-2) [n:127.0.0.1:37548_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 580425 INFO (jetty-launcher-615-thread-2) [n:127.0.0.1:37548_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37548_solr
[junit4] 2> 580425 INFO (jetty-launcher-615-thread-2) [n:127.0.0.1:37548_solr ] o.a.s.c.Overseer Overseer (id=98848518611533830-127.0.0.1:37548_solr-n_0000000000) starting
[junit4] 2> 580706 INFO (jetty-launcher-615-thread-2) [n:127.0.0.1:37548_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37548_solr
[junit4] 2> 580754 INFO (jetty-launcher-615-thread-1) [n:127.0.0.1:38739_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 580767 INFO (zkCallback-627-thread-1-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 580787 INFO (jetty-launcher-615-thread-1) [n:127.0.0.1:38739_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 580789 INFO (jetty-launcher-615-thread-1) [n:127.0.0.1:38739_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38739_solr
[junit4] 2> 580825 INFO (zkCallback-627-thread-2-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 580893 INFO (zkCallback-626-thread-1-processing-n:127.0.0.1:38739_solr) [n:127.0.0.1:38739_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 581507 INFO (jetty-launcher-615-thread-1) [n:127.0.0.1:38739_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38739.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6cad6490
[junit4] 2> 581628 INFO (jetty-launcher-615-thread-1) [n:127.0.0.1:38739_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38739.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6cad6490
[junit4] 2> 581663 INFO (jetty-launcher-615-thread-1) [n:127.0.0.1:38739_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38739.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6cad6490
[junit4] 2> 581664 INFO (jetty-launcher-615-thread-1) [n:127.0.0.1:38739_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_D68D6E87E44560AB-001/tempDir-001/node1/.
[junit4] 2> 581701 INFO (jetty-launcher-615-thread-2) [n:127.0.0.1:37548_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37548.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6cad6490
[junit4] 2> 582015 INFO (jetty-launcher-615-thread-2) [n:127.0.0.1:37548_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37548.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6cad6490
[junit4] 2> 582016 INFO (jetty-launcher-615-thread-2) [n:127.0.0.1:37548_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37548.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6cad6490
[junit4] 2> 582049 INFO (jetty-launcher-615-thread-2) [n:127.0.0.1:37548_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_D68D6E87E44560AB-001/tempDir-001/node2/.
[junit4] 2> 582826 INFO (SUITE-TestTlogReplica-seed#[D68D6E87E44560AB]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 582826 INFO (SUITE-TestTlogReplica-seed#[D68D6E87E44560AB]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34377/solr ready
[junit4] 2> 582828 INFO (SUITE-TestTlogReplica-seed#[D68D6E87E44560AB]-worker) [ ] o.a.s.c.TestTlogReplica Using legacyCloud?: false
[junit4] 2> 583254 INFO (qtp348955250-2460) [n:127.0.0.1:37548_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 583293 INFO (qtp348955250-2460) [n:127.0.0.1:37548_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=39
[junit4] 2> 583438 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testOutOfOrderDBQWithInPlaceUpdates
[junit4] 2> 583545 INFO (qtp348955250-2461) [n:127.0.0.1:37548_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=0&collection.configName=conf&maxShardsPerNode=100&name=tlog_replica_test_out_of_order_db_qwith_in_place_updates&nrtReplicas=0&action=CREATE&numShards=1&tlogReplicas=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 583559 INFO (OverseerThreadFactory-745-thread-1-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.CreateCollectionCmd Create collection tlog_replica_test_out_of_order_db_qwith_in_place_updates
[junit4] 2> 583727 INFO (OverseerStateUpdate-98848518611533830-127.0.0.1:37548_solr-n_0000000000) [n:127.0.0.1:37548_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"tlog_replica_test_out_of_order_db_qwith_in_place_updates",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:38739/solr",
[junit4] 2> "type":"TLOG"}
[junit4] 2> 583762 INFO (OverseerStateUpdate-98848518611533830-127.0.0.1:37548_solr-n_0000000000) [n:127.0.0.1:37548_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"tlog_replica_test_out_of_order_db_qwith_in_place_updates",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:37548/solr",
[junit4] 2> "type":"TLOG"}
[junit4] 2> 584208 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2&action=CREATE&numShards=1&collection=tlog_replica_test_out_of_order_db_qwith_in_place_updates&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 584280 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 584372 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1&action=CREATE&numShards=1&collection=tlog_replica_test_out_of_order_db_qwith_in_place_updates&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 584372 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 584457 INFO (zkCallback-627-thread-2-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 584457 INFO (zkCallback-627-thread-1-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 584676 INFO (zkCallback-627-thread-1-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 584676 INFO (zkCallback-626-thread-1-processing-n:127.0.0.1:38739_solr) [n:127.0.0.1:38739_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 584676 INFO (zkCallback-627-thread-2-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 585473 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 585511 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.s.IndexSchema [tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] Schema name=minimal
[junit4] 2> 585513 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 585513 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2' using configuration from collection tlog_replica_test_out_of_order_db_qwith_in_place_updates, trusted=true
[junit4] 2> 585564 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37548.solr.core.tlog_replica_test_out_of_order_db_qwith_in_place_updates.shard1.replica_t2' (registry 'solr.core.tlog_replica_test_out_of_order_db_qwith_in_place_updates.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6cad6490
[junit4] 2> 585564 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 585564 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.c.SolrCore [[tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_D68D6E87E44560AB-001/tempDir-001/node2/tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_D68D6E87E44560AB-001/tempDir-001/node2/./tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2/data/]
[junit4] 2> 585576 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 585692 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.s.IndexSchema [tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] Schema name=minimal
[junit4] 2> 585745 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 585746 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1' using configuration from collection tlog_replica_test_out_of_order_db_qwith_in_place_updates, trusted=true
[junit4] 2> 585779 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38739.solr.core.tlog_replica_test_out_of_order_db_qwith_in_place_updates.shard1.replica_t1' (registry 'solr.core.tlog_replica_test_out_of_order_db_qwith_in_place_updates.shard1.replica_t1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6cad6490
[junit4] 2> 585779 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 585779 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.SolrCore [[tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_D68D6E87E44560AB-001/tempDir-001/node1/tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_D68D6E87E44560AB-001/tempDir-001/node1/./tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1/data/]
[junit4] 2> 586125 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 586125 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 586126 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 586126 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 586128 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@666b4fde[tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] main]
[junit4] 2> 586146 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 586146 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 586147 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 586147 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581576305191682048
[junit4] 2> 586220 INFO (searcherExecutor-751-thread-1-processing-n:127.0.0.1:38739_solr x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1 s:shard1 c:tlog_replica_test_out_of_order_db_qwith_in_place_updates r:core_node3) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.SolrCore [tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] Registered new searcher Searcher@666b4fde[tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 586220 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 586220 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 586257 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 586257 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 586257 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 586291 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@37530f00[tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] main]
[junit4] 2> 586308 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 586308 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 586309 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 586309 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581576305361551360
[junit4] 2> 586345 INFO (searcherExecutor-750-thread-1-processing-n:127.0.0.1:37548_solr x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2 s:shard1 c:tlog_replica_test_out_of_order_db_qwith_in_place_updates r:core_node4) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.c.SolrCore [tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] Registered new searcher Searcher@37530f00[tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 586417 INFO (zkCallback-626-thread-1-processing-n:127.0.0.1:38739_solr) [n:127.0.0.1:38739_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 586417 INFO (zkCallback-627-thread-2-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 586417 INFO (zkCallback-627-thread-3-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 586787 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 586787 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 586787 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:38739/solr/tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1/
[junit4] 2> 586787 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1 url=https://127.0.0.1:38739/solr START replicas=[https://127.0.0.1:37548/solr/tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2/] nUpdates=100
[junit4] 2> 586819 INFO (qtp348955250-2459) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 587115 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1 url=https://127.0.0.1:38739/solr DONE. We have no versions. sync failed.
[junit4] 2> 587115 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 587115 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 587115 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 587127 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.ZkController tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1 stopping background replication from leader
[junit4] 2> 587129 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:38739/solr/tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1/ shard1
[junit4] 2> 587306 INFO (zkCallback-626-thread-1-processing-n:127.0.0.1:38739_solr) [n:127.0.0.1:38739_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 587318 INFO (zkCallback-627-thread-2-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 587318 INFO (zkCallback-627-thread-3-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 587343 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 587343 INFO (qtp1082975349-2451) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1&action=CREATE&numShards=1&collection=tlog_replica_test_out_of_order_db_qwith_in_place_updates&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2971
[junit4] 2> 587365 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.c.ZkController tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2 starting background replication from leader
[junit4] 2> 587366 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 587473 INFO (zkCallback-626-thread-1-processing-n:127.0.0.1:38739_solr) [n:127.0.0.1:38739_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 587474 INFO (zkCallback-627-thread-3-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 587474 INFO (zkCallback-627-thread-2-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 587546 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 587546 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 587548 INFO (qtp348955250-2463) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2&action=CREATE&numShards=1&collection=tlog_replica_test_out_of_order_db_qwith_in_place_updates&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=3340
[junit4] 2> 587582 INFO (indexFetcher-761-thread-1) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.h.IndexFetcher Updated masterUrl to https://127.0.0.1:38739/solr/tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1/
[junit4] 2> 587675 INFO (qtp348955250-2461) [n:127.0.0.1:37548_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 587731 INFO (qtp1082975349-2445) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 587731 INFO (indexFetcher-761-thread-1) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 587731 INFO (indexFetcher-761-thread-1) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 587731 INFO (indexFetcher-761-thread-1) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 587731 INFO (indexFetcher-761-thread-1) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 587731 INFO (indexFetcher-761-thread-1) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 587766 INFO (indexFetcher-761-thread-1) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@553a7abd[tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] main]
[junit4] 2> 587767 INFO (searcherExecutor-750-thread-1-processing-n:127.0.0.1:37548_solr x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2 s:shard1 c:tlog_replica_test_out_of_order_db_qwith_in_place_updates r:core_node4) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.c.SolrCore [tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] Registered new searcher Searcher@553a7abd[tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 587781 INFO (zkCallback-626-thread-1-processing-n:127.0.0.1:38739_solr) [n:127.0.0.1:38739_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 587782 INFO (zkCallback-627-thread-2-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 587782 INFO (zkCallback-627-thread-3-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 587783 INFO (OverseerCollectionConfigSetProcessor-98848518611533830-127.0.0.1:37548_solr-n_0000000000) [n:127.0.0.1:37548_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 588690 INFO (qtp348955250-2461) [n:127.0.0.1:37548_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=0&collection.configName=conf&maxShardsPerNode=100&name=tlog_replica_test_out_of_order_db_qwith_in_place_updates&nrtReplicas=0&action=CREATE&numShards=1&tlogReplicas=2&wt=javabin&version=2} status=0 QTime=5144
[junit4] 2> 588854 INFO (qtp348955250-2460) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:38739/solr/tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1/&wt=javabin&version=2}{add=[1 (1)]} 0 1
[junit4] 2> 588855 INFO (qtp348955250-2460) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&_version_=3&distrib.from=https://127.0.0.1:38739/solr/tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1/&wt=javabin&version=2}{deleteByQuery=inplace_updatable_int:5 (-3)} 0 0
[junit4] 2> 588872 INFO (qtp348955250-2460) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.inplace.prevversion=1&distrib.from=https://127.0.0.1:38739/solr/tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1/&wt=javabin&version=2&distrib.inplace.update=true}{add=[1 (2)]} 0 16
[junit4] 2> 588876 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.c.ChaosMonkey monkey: kill jetty! 38739
[junit4] 2> 588947 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@20e7dc06{SSL,[ssl, http/1.1]}{127.0.0.1:0}
[junit4] 2> 588962 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1746671959
[junit4] 2> 588983 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 588983 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@57631470: rootName = solr_38739, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@2fb95873
[junit4] 2> 589001 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 589001 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@711658c0: rootName = solr_38739, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@7b01ee03
[junit4] 2> 589021 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 589021 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@47cc60f6: rootName = solr_38739, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@6617a0f8
[junit4] 2> 589022 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 589059 INFO (coreCloseExecutor-762-thread-1) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.c.SolrCore [tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] CLOSING SolrCore org.apache.solr.core.SolrCore@70bbb25c
[junit4] 2> 589059 INFO (coreCloseExecutor-762-thread-1) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.tlog_replica_test_out_of_order_db_qwith_in_place_updates.shard1.replica_t1, tag=1891349084
[junit4] 2> 589060 INFO (coreCloseExecutor-762-thread-1) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6149ed5a: rootName = solr_38739, domain = solr.core.tlog_replica_test_out_of_order_db_qwith_in_place_updates.shard1.replica_t1, service url = null, agent id = null] for registry solr.core.tlog_replica_test_out_of_order_db_qwith_in_place_updates.shard1.replica_t1 / com.codahale.metrics.MetricRegistry@227bc600
[junit4] 2> 589254 INFO (zkCallback-626-thread-1-processing-n:127.0.0.1:38739_solr) [n:127.0.0.1:38739_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 589255 INFO (zkCallback-627-thread-2-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 589258 INFO (zkCallback-627-thread-3-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 589262 INFO (zkCallback-634-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 589385 INFO (coreCloseExecutor-762-thread-1) [n:127.0.0.1:38739_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node3 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.tlog_replica_test_out_of_order_db_qwith_in_place_updates.shard1.leader, tag=1891349084
[junit4] 2> 589419 INFO (zkCallback-627-thread-3-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 589420 INFO (zkCallback-627-thread-2-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 589435 INFO (zkCallback-634-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 589583 INFO (zkCallback-627-thread-3-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [1])
[junit4] 2> 589583 INFO (zkCallback-627-thread-1-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [1])
[junit4] 2> 589595 INFO (zkCallback-634-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [1])
[junit4] 2> 590926 WARN (zkCallback-626-thread-1-processing-n:127.0.0.1:38739_solr) [n:127.0.0.1:38739_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 590927 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@74db45c0{/solr,null,UNAVAILABLE}
[junit4] 2> 590943 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.c.ChaosMonkey monkey: stop jetty! 38739
[junit4] 2> 590963 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.c.ChaosMonkey monkey: starting jetty! 38739
[junit4] 2> 590964 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 590994 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7cbfdff7{/solr,null,AVAILABLE}
[junit4] 2> 590995 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7ea87654{SSL,[ssl, http/1.1]}{127.0.0.1:38739}
[junit4] 2> 590995 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.e.j.s.Server Started @607802ms
[junit4] 2> 590995 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38739}
[junit4] 2> 590995 ERROR (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 590995 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 590995 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 590995 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 591009 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-18T06:38:26.560Z
[junit4] 2> 591146 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 591231 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34377/solr
[junit4] 2> 591328 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [n:127.0.0.1:38739_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 591361 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [n:127.0.0.1:38739_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 591362 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [n:127.0.0.1:38739_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38739_solr
[junit4] 2> 591363 INFO (zkCallback-627-thread-3-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 591398 INFO (zkCallback-639-thread-1-processing-n:127.0.0.1:38739_solr) [n:127.0.0.1:38739_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 591414 INFO (zkCallback-634-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 591564 INFO (zkCallback-627-thread-3-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 591564 INFO (zkCallback-627-thread-1-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 591576 INFO (zkCallback-634-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json] for collection [tlog_replica_test_out_of_order_db_qwith_in_place_updates] has occurred - updating... (live nodes size: [2])
[junit4] 2> 591929 INFO (zkCallback-627-thread-2-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:37548/solr/tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2/
[junit4] 2> 591930 INFO (zkCallback-627-thread-2-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2 url=https://127.0.0.1:37548/solr START replicas=[https://127.0.0.1:38739/solr/tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1/] nUpdates=100
[junit4] 2> 592637 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [n:127.0.0.1:38739_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38739.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6cad6490
[junit4] 2> 592674 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [n:127.0.0.1:38739_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38739.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6cad6490
[junit4] 2> 592674 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [n:127.0.0.1:38739_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38739.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6cad6490
[junit4] 2> 592676 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [n:127.0.0.1:38739_solr ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_D68D6E87E44560AB-001/tempDir-001/node1/.
[junit4] 2> 592676 INFO (TEST-TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates-seed#[D68D6E87E44560AB]) [n:127.0.0.1:38739_solr ] o.a.s.c.CorePropertiesLocator Cores are: [tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1]
[junit4] 2> 592693 INFO (coreLoadExecutor-769-thread-1-processing-n:127.0.0.1:38739_solr) [n:127.0.0.1:38739_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 592767 INFO (coreLoadExecutor-769-thread-1-processing-n:127.0.0.1:38739_solr) [n:127.0.0.1:38739_solr ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 592806 INFO (coreLoadExecutor-769-thread-1-processing-n:127.0.0.1:38739_solr) [n:127.0.0.1:38739_solr ] o.a.s.s.IndexSchema [tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1] Schema name=minimal
[junit4] 2> 592820 INFO (coreLoadExecutor-769-thread-1-processing-n:127.0.0.1:38739_solr) [n:127.0.0.1:38739_solr ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 592856 WARN (zkCallback-627-thread-2-processing-n:127.0.0.1:37548_solr) [n:127.0.0.1:37548_solr c:tlog_replica_test_out_of_order_db_qwith_in_place_updates s:shard1 r:core_node4 x:tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t2 url=https://127.0.0.1:37548/solr got a 503 from https://127.0.0.1:38739/solr/tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1/, counting as success
[junit4] 2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:38739/solr/tlog_replica_test_out_of_order_db_qwith_in_place_updates_shard1_replica_t1: Expected mime type application/octet-stream but got application/json. {
[junit4] 2> "error":{
[junit4] 2> "metadata":[
[junit4] 2> "error-class","org.apache.solr.common.SolrException",
[junit4] 2> "root-error-class","org.apache.solr.common.SolrException"],
[junit4] 2> "msg":"SolrCore is loading",
[junit4] 2> "code":503}}
[junit4] 2>
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:590)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
[junit4] 2> at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
[junit4] 2> at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWork
[...truncated too long message...]
0.0.1:38997_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/falsepredicate/state.json] for collection [falsepredicate] has occurred - updating... (live nodes size: [3])
[junit4] 2> 159305 INFO (zkCallback-46-thread-5-processing-n:127.0.0.1:38997_solr) [n:127.0.0.1:38997_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/delayed/state.json] for collection [delayed] has occurred - updating... (live nodes size: [3])
[junit4] 2> 159305 INFO (zkCallback-46-thread-6-processing-n:127.0.0.1:38997_solr) [n:127.0.0.1:38997_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/delayed/state.json] for collection [delayed] has occurred - updating... (live nodes size: [3])
[junit4] 2> 159317 INFO (zkCallback-46-thread-3-processing-n:127.0.0.1:38997_solr) [n:127.0.0.1:38997_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testcollection/state.json] for collection [testcollection] has occurred - updating... (live nodes size: [3])
[junit4] 2> 159317 INFO (zkCallback-46-thread-1-processing-n:127.0.0.1:38997_solr) [n:127.0.0.1:38997_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/falsepredicate/state.json] for collection [falsepredicate] has occurred - updating... (live nodes size: [3])
[junit4] 2> 159353 INFO (coreCloseExecutor-139-thread-1) [n:127.0.0.1:34220_solr c:falsepredicate s:shard3 r:core_node7 x:falsepredicate_shard3_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.falsepredicate.shard3.leader, tag=1899405215
[junit4] 2> 159354 INFO (coreCloseExecutor-138-thread-1) [n:127.0.0.1:41639_solr c:currentstate s:shard1 r:core_node2 x:currentstate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.currentstate.shard1.leader, tag=435480902
[junit4] 2> 159365 INFO (coreCloseExecutor-139-thread-2) [n:127.0.0.1:34220_solr c:testcollection s:shard1 r:core_node3 x:testcollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testcollection.shard1.replica_n1, tag=1441291500
[junit4] 2> 159366 INFO (coreCloseExecutor-139-thread-2) [n:127.0.0.1:34220_solr c:testcollection s:shard1 r:core_node3 x:testcollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3446c670: rootName = solr_34220, domain = solr.core.testcollection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.testcollection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@1ac79d14
[junit4] 2> 159369 INFO (coreCloseExecutor-138-thread-2) [n:127.0.0.1:41639_solr c:falsepredicate s:shard4 r:core_node8 x:falsepredicate_shard4_replica_n6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.falsepredicate.shard4.replica_n6, tag=458858428
[junit4] 2> 159369 INFO (coreCloseExecutor-138-thread-2) [n:127.0.0.1:41639_solr c:falsepredicate s:shard4 r:core_node8 x:falsepredicate_shard4_replica_n6] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2f510806: rootName = solr_41639, domain = solr.core.falsepredicate.shard4.replica_n6, service url = null, agent id = null] for registry solr.core.falsepredicate.shard4.replica_n6 / com.codahale.metrics.MetricRegistry@5c862083
[junit4] 2> 159467 INFO (coreCloseExecutor-139-thread-2) [n:127.0.0.1:34220_solr c:testcollection s:shard1 r:core_node3 x:testcollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard1.leader, tag=1441291500
[junit4] 2> 159611 INFO (coreCloseExecutor-138-thread-3) [n:127.0.0.1:41639_solr c:testcollection s:shard3 r:core_node7 x:testcollection_shard3_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testcollection.shard3.replica_n4, tag=1728062708
[junit4] 2> 159611 INFO (coreCloseExecutor-138-thread-3) [n:127.0.0.1:41639_solr c:testcollection s:shard3 r:core_node7 x:testcollection_shard3_replica_n4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@682cf7d9: rootName = solr_41639, domain = solr.core.testcollection.shard3.replica_n4, service url = null, agent id = null] for registry solr.core.testcollection.shard3.replica_n4 / com.codahale.metrics.MetricRegistry@233c52e8
[junit4] 2> 159739 INFO (coreCloseExecutor-138-thread-3) [n:127.0.0.1:41639_solr c:testcollection s:shard3 r:core_node7 x:testcollection_shard3_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard3.leader, tag=1728062708
[junit4] 2> 159739 INFO (coreCloseExecutor-138-thread-2) [n:127.0.0.1:41639_solr c:falsepredicate s:shard4 r:core_node8 x:falsepredicate_shard4_replica_n6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.falsepredicate.shard4.leader, tag=458858428
[junit4] 2> 159749 INFO (zkCallback-22-thread-1-processing-n:127.0.0.1:41639_solr) [n:127.0.0.1:41639_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 160101 INFO (zkCallback-21-thread-4-processing-n:127.0.0.1:34220_solr) [n:127.0.0.1:34220_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 160126 INFO (jetty-closer-3-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3eaf9326{/solr,null,UNAVAILABLE}
[junit4] 2> 160163 INFO (jetty-closer-3-thread-2) [ ] o.a.s.c.Overseer Overseer (id=98848943693037577-127.0.0.1:34220_solr-n_0000000000) closing
[junit4] 2> 160171 INFO (OverseerStateUpdate-98848943693037577-127.0.0.1:34220_solr-n_0000000000) [n:127.0.0.1:34220_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:34220_solr
[junit4] 2> 160450 WARN (OverseerAutoScalingTriggerThread-98848943693037577-127.0.0.1:34220_solr-n_0000000000) [n:127.0.0.1:34220_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 160611 WARN (zkCallback-22-thread-1-processing-n:127.0.0.1:41639_solr) [n:127.0.0.1:41639_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 160613 INFO (jetty-closer-3-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@47f8f65{/solr,null,UNAVAILABLE}
[junit4] 2> 160614 INFO (zkCallback-21-thread-4-processing-n:127.0.0.1:34220_solr) [n:127.0.0.1:34220_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 162193 WARN (zkCallback-21-thread-4-processing-n:127.0.0.1:34220_solr) [n:127.0.0.1:34220_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 162193 INFO (jetty-closer-3-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@39bc1944{/solr,null,UNAVAILABLE}
[junit4] 2> 162229 ERROR (SUITE-TestCollectionStateWatchers-seed#[DA5E219E64092448]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 162230 INFO (SUITE-TestCollectionStateWatchers-seed#[DA5E219E64092448]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43001 43001
[junit4] 2> 162477 INFO (Thread-1) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43001 43001
[junit4] 2> 162492 WARN (Thread-1) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/aliases.json
[junit4] 2> 7 /solr/configs/config
[junit4] 2> 6 /solr/security.json
[junit4] 2> 4 /solr/collections/stateformat1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 65 /solr/collections/falsepredicate/state.json
[junit4] 2> 51 /solr/collections/testcollection/state.json
[junit4] 2> 21 /solr/collections/waitforstate/state.json
[junit4] 2> 11 /solr/collections/delayed/state.json
[junit4] 2> 9 /solr/collections/stateformat1/state.json
[junit4] 2> 8 /solr/collections/currentstate/state.json
[junit4] 2> 8 /solr/clusterprops.json
[junit4] 2> 8 /solr/clusterstate.json
[junit4] 2> 6 /solr/collections/tobedeleted/state.json
[junit4] 2> 3 /solr/overseer_elect/election/98848943693037577-127.0.0.1:34220_solr-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/collections
[junit4] 2> 6 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-solrj/test/J1/temp/solr.common.cloud.TestCollectionStateWatchers_DA5E219E64092448-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {}, locale=ar-LY, timezone=America/Glace_Bay
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=86368352,total=310902784
[junit4] 2> NOTE: All tests run in this JVM: [TestValidatingJsonMap, TestCollectionStateWatchers]
[junit4] Completed [17/145 (1!)] on J1 in 159.93s, 9 tests, 1 failure <<< FAILURES!
[...truncated 46675 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 184 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/184/
10 tests failed.
FAILED: org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest
Error Message:
Stack Trace:
java.lang.NullPointerException
at __randomizedtesting.SeedInfo.seed([5952CD777F534844:F432797C626CE031]:0)
at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
at org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest(DeleteReplicaTest.java:68)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration
Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:40137_solr wasn't created
Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:40137_solr wasn't created
at __randomizedtesting.SeedInfo.seed([5952CD777F534844:41E8457B716685AB]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration(TriggerIntegrationTest.java:909)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.handler.TestBlobHandler.doBlobHandlerTest
Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:38240/ha
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:38240/ha
at __randomizedtesting.SeedInfo.seed([5952CD777F534844:B993EF25C4BF3EB6]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:637)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:418)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
... 44 more
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestBlobHandler
Error Message:
119 threads leaked from SUITE scope at org.apache.solr.handler.TestBlobHandler: 1) Thread[id=10600, name=zkCallback-2888-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=10603, name=Scheduler-286463277, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=10617, name=qtp845990262-10617, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=10737, name=searcherExecutor-3580-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=10662, name=zkCallback-2901-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=10586, name=qtp400111427-10586, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=10689, name=zkCallback-2901-thread-6, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=10710, name=zkCallback-2894-thread-2-processing-n:127.0.0.1:38240_ha-EventThread, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 9) Thread[id=10633, name=Scheduler-119567986, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=10610, name=qtp845990262-10610, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=10615, name=qtp845990262-10615, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=10593, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=10642, name=qtp398765038-10642-acceptor-0@3930d604-ServerConnector@33a1fdbe{HTTP/1.1,[http/1.1]}{127.0.0.1:49107}, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=10605, name=watches-2889-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=10660, name=TEST-TestBlobHandler.doBlobHandlerTest-seed#[5952CD777F534844]-SendThread(127.0.0.1:49437), state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230) at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1246) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1170) 16) Thread[id=10688, name=zkCallback-2901-thread-5, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=10634, name=watches-2895-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=10584, name=qtp400111427-10584, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=10557, name=org.eclipse.jetty.server.session.HashSessionManager@4e631201Timer, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=10643, name=qtp398765038-10643, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=10745, name=zkCallback-2894-thread-8, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=10572, name=Scheduler-179602796, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=10570, name=Thread-1843, state=WAITING, group=TGRP-TestBlobHandler] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1673) 24) Thread[id=10550, name=qtp1907887861-10550, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=10669, name=zkCallback-2901-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=10675, name=zkCallback-2894-thread-5, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=10622, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=10727, name=ScheduledTrigger-3573-thread-3, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=10607, name=searcherExecutor-3549-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=10630, name=Thread-1861, state=WAITING, group=TGRP-TestBlobHandler] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1673) 31) Thread[id=10706, name=zkCallback-2882-thread-2-processing-n:127.0.0.1:47368_ha-SendThread(127.0.0.1:49437), state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230) at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1246) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1170) 32) Thread[id=10655, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=10703, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=10549, name=qtp1907887861-10549, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=10714, name=coreZkRegister-3554-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=10598, name=TEST-TestBlobHandler.doBlobHandlerTest-seed#[5952CD777F534844]-SendThread(127.0.0.1:49437), state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 37) Thread[id=10589, name=org.eclipse.jetty.server.session.HashSessionManager@3a3a7bfeTimer, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 38) Thread[id=10646, name=qtp398765038-10646, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=10693, name=zkCallback-2888-thread-6, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 40) Thread[id=10645, name=qtp398765038-10645, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 41) Thread[id=10561, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 42) Thread[id=10644, name=qtp398765038-10644, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 43) Thread[id=10614, name=qtp845990262-10614, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 44) Thread[id=10551, name=qtp1907887861-10551-acceptor-0@4208a33a-ServerConnector@7787c4d8{HTTP/1.1,[http/1.1]}{127.0.0.1:47368}, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 45) Thread[id=10698, name=zkCallback-2894-thread-7, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 46) Thread[id=10554, name=qtp1907887861-10554, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 47) Thread[id=10679, name=zkCallback-2888-thread-5, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 48) Thread[id=10713, name=coreZkRegister-3532-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 49) Thread[id=10746, name=zkCallback-2882-thread-7, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 50) Thread[id=10648, name=org.eclipse.jetty.server.session.HashSessionManager@18f00974Timer, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 51) Thread[id=10721, name=recoveryExecutor-2899-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 52) Thread[id=10588, name=qtp400111427-10588, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 53) Thread[id=10685, name=zkCallback-2882-thread-3, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 54) Thread[id=10647, name=qtp398765038-10647, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 55) Thread[id=10636, name=searcherExecutor-3560-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 56) Thread[id=10668, name=watches-2902-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 57) Thread[id=10576, name=watches-2883-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 58) Thread[id=10629, name=zkCallback-2894-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 59) Thread[id=10656, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 60) Thread[id=10676, name=zkCallback-2888-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 61) Thread[id=10623, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 62) Thread[id=10652, name=recoveryExecutor-2892-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 63) Thread[id=10695, name=zkCallback-2882-thread-6, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 64) Thread[id=10708, name=OverseerThreadFactory-3575-thread-1, state=WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 65) Thread[id=10568, name=zkCallback-2882-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 66) Thread[id=10601, name=Thread-1852, state=WAITING, group=TGRP-TestBlobHandler] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1673) 67) Thread[id=10618, name=org.eclipse.jetty.server.session.HashSessionManager@774275Timer, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 68) Thread[id=10747, name=zkCallback-2901-thread-7, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 69) Thread[id=10599, name=TEST-TestBlobHandler.doBlobHandlerTest-seed#[5952CD777F534844]-EventThread, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 70) Thread[id=10694, name=zkCallback-2882-thread-5, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 71) Thread[id=10702, name=OverseerCollectionConfigSetProcessor-98840392147927052-127.0.0.1:34756_ha-n_0000000002, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 72) Thread[id=10674, name=zkCallback-2894-thread-4, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 73) Thread[id=10664, name=Thread-1870, state=WAITING, group=TGRP-TestBlobHandler] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1673) 74) Thread[id=10718, name=searcherExecutor-3571-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 75) Thread[id=10672, name=zkCallback-2894-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 76) Thread[id=10580, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 77) Thread[id=10709, name=zkCallback-2894-thread-2-processing-n:127.0.0.1:38240_ha-SendThread(127.0.0.1:49437), state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 78) Thread[id=10582, name=qtp400111427-10582, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 79) Thread[id=10667, name=Scheduler-336929368, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 80) Thread[id=10671, name=zkCallback-2901-thread-4, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 81) Thread[id=10613, name=qtp845990262-10613, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 82) Thread[id=10640, name=qtp398765038-10640, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 83) Thread[id=10641, name=qtp398765038-10641, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 84) Thread[id=10585, name=qtp400111427-10585, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 85) Thread[id=10744, name=zkCallback-2888-thread-8, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 86) Thread[id=10637, name=updateExecutor-2891-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 87) Thread[id=10569, name=zkCallback-2882-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 88) Thread[id=10704, name=OverseerAutoScalingTriggerThread-98840392147927052-127.0.0.1:34756_ha-n_0000000002, state=WAITING, group=Overseer autoscaling triggers] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:161) at java.lang.Thread.run(Thread.java:748) 89) Thread[id=10686, name=zkCallback-2882-thread-4, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 90) Thread[id=10743, name=zkCallback-2888-thread-7, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 91) Thread[id=10616, name=qtp845990262-10616, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 92) Thread[id=10578, name=searcherExecutor-3538-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 93) Thread[id=10583, name=qtp400111427-10583-acceptor-0@2e883f63-ServerConnector@77f569cf{HTTP/1.1,[http/1.1]}{127.0.0.1:34756}, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 94) Thread[id=10678, name=zkCallback-2888-thread-4, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 95) Thread[id=10552, name=qtp1907887861-10552, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 96) Thread[id=10609, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 97) Thread[id=10712, name=coreZkRegister-3532-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 98) Thread[id=10556, name=qtp1907887861-10556, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method)
[...truncated too long message...]
r-208-thread-1) [n:127.0.0.1:32897_solr c:currentstate s:shard1 r:core_node2 x:currentstate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.currentstate.shard1.replica_n1, tag=1941638547
[junit4] 2> 331499 INFO (coreCloseExecutor-208-thread-1) [n:127.0.0.1:32897_solr c:currentstate s:shard1 r:core_node2 x:currentstate_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@399405ee: rootName = solr_32897, domain = solr.core.currentstate.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.currentstate.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@6fb2f8d7
[junit4] 2> 331539 INFO (coreCloseExecutor-207-thread-1) [n:127.0.0.1:50145_solr c:delayed s:shard1 r:core_node2 x:delayed_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1e9c2cb7: rootName = solr_50145, domain = solr.core.delayed.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.delayed.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@7cbabef8
[junit4] 2> 331702 INFO (jetty-closer-45-thread-1) [ ] o.a.s.c.Overseer Overseer (id=98840465108107271-127.0.0.1:34535_solr-n_0000000001) closing
[junit4] 2> 331711 INFO (OverseerStateUpdate-98840465108107271-127.0.0.1:34535_solr-n_0000000001) [n:127.0.0.1:34535_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:34535_solr
[junit4] 2> 331737 INFO (coreCloseExecutor-207-thread-1) [n:127.0.0.1:50145_solr c:delayed s:shard1 r:core_node2 x:delayed_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.delayed.shard1.leader, tag=1582971442
[junit4] 2> 331751 WARN (OverseerAutoScalingTriggerThread-98840465108107271-127.0.0.1:34535_solr-n_0000000001) [n:127.0.0.1:34535_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 331796 INFO (coreCloseExecutor-208-thread-1) [n:127.0.0.1:32897_solr c:currentstate s:shard1 r:core_node2 x:currentstate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.currentstate.shard1.leader, tag=1941638547
[junit4] 2> 331802 INFO (coreCloseExecutor-208-thread-5) [n:127.0.0.1:32897_solr c:stateformat1 s:shard1 r:core_node2 x:stateformat1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.stateformat1.shard1.replica_n1, tag=352433011
[junit4] 2> 331802 INFO (coreCloseExecutor-208-thread-5) [n:127.0.0.1:32897_solr c:stateformat1 s:shard1 r:core_node2 x:stateformat1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2298050d: rootName = solr_32897, domain = solr.core.stateformat1.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.stateformat1.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@208b4dcf
[junit4] 2> 331844 INFO (coreCloseExecutor-208-thread-4) [n:127.0.0.1:32897_solr c:waitforstate s:shard1 r:core_node2 x:waitforstate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.waitforstate.shard1.leader, tag=370242055
[junit4] 2> 331844 INFO (coreCloseExecutor-208-thread-3) [n:127.0.0.1:32897_solr c:testcollection s:shard1 r:core_node3 x:testcollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard1.leader, tag=1733924543
[junit4] 2> 331845 INFO (coreCloseExecutor-208-thread-5) [n:127.0.0.1:32897_solr c:stateformat1 s:shard1 r:core_node2 x:stateformat1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.stateformat1.shard1.leader, tag=352433011
[junit4] 2> 331851 INFO (coreCloseExecutor-208-thread-2) [n:127.0.0.1:32897_solr c:falsepredicate s:shard1 r:core_node3 x:falsepredicate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.falsepredicate.shard1.leader, tag=1734088858
[junit4] 2> 331888 WARN (zkCallback-84-thread-1-processing-n:127.0.0.1:42506_solr) [n:127.0.0.1:42506_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 331903 INFO (zkCallback-60-thread-4-processing-n:127.0.0.1:34535_solr) [n:127.0.0.1:34535_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 331903 INFO (zkCallback-100-thread-3-processing-n:127.0.0.1:32897_solr) [n:127.0.0.1:32897_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 331903 INFO (zkCallback-94-thread-1-processing-n:127.0.0.1:50145_solr) [n:127.0.0.1:50145_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 331947 INFO (jetty-closer-45-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@36fe44be{/solr,null,UNAVAILABLE}
[junit4] 2> 331949 INFO (zkCallback-60-thread-4-processing-n:127.0.0.1:34535_solr) [n:127.0.0.1:34535_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 331973 INFO (zkCallback-100-thread-4-processing-n:127.0.0.1:32897_solr) [n:127.0.0.1:32897_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 332008 INFO (zkCallback-100-thread-3-processing-n:127.0.0.1:32897_solr) [n:127.0.0.1:32897_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 332008 INFO (zkCallback-100-thread-4-processing-n:127.0.0.1:32897_solr) [n:127.0.0.1:32897_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:32897_solr
[junit4] 2> 332040 INFO (jetty-closer-45-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@30c4dc51{/solr,null,UNAVAILABLE}
[junit4] 2> 332252 INFO (jetty-closer-45-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6cbbfc8{/solr,null,UNAVAILABLE}
[junit4] 2> 333451 WARN (zkCallback-94-thread-1-processing-n:127.0.0.1:50145_solr) [n:127.0.0.1:50145_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 333452 INFO (jetty-closer-45-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@76ea34fa{/solr,null,UNAVAILABLE}
[junit4] 2> 333460 ERROR (SUITE-TestCollectionStateWatchers-seed#[F924DF6DC1F87583]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 333460 INFO (SUITE-TestCollectionStateWatchers-seed#[F924DF6DC1F87583]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34710 34710
[junit4] 2> 333554 INFO (Thread-25) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34710 34710
[junit4] 2> 333554 WARN (Thread-25) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/aliases.json
[junit4] 2> 8 /solr/clusterprops.json
[junit4] 2> 7 /solr/security.json
[junit4] 2> 7 /solr/configs/config
[junit4] 2> 4 /solr/collections/stateformat1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 74 /solr/collections/falsepredicate/state.json
[junit4] 2> 53 /solr/collections/testcollection/state.json
[junit4] 2> 21 /solr/collections/waitforstate/state.json
[junit4] 2> 8 /solr/collections/currentstate/state.json
[junit4] 2> 8 /solr/clusterstate.json
[junit4] 2> 7 /solr/collections/delayed/state.json
[junit4] 2> 6 /solr/collections/tobedeleted/state.json
[junit4] 2> 5 /solr/overseer_elect/election/98840465108107271-127.0.0.1:34535_solr-n_0000000001
[junit4] 2> 5 /solr/collections/stateformat1/state.json
[junit4] 2> 2 /solr/autoscaling.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/collections
[junit4] 2> 5 /solr/live_nodes
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/overseer/collection-queue-work
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-solrj/test/J1/temp/solr.common.cloud.TestCollectionStateWatchers_F924DF6DC1F87583-001
[junit4] 2> Oct 16, 2017 8:32:36 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1352, maxMBSortInHeap=7.413950038589327, sim=RandomSimilarity(queryNorm=true): {}, locale=be, timezone=Mexico/BajaSur
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=146280664,total=351272960
[junit4] 2> NOTE: All tests run in this JVM: [CloudSolrClientBuilderTest, AddEvaluatorTest, TestSuggesterResponse, SolrDocumentTest, SolrExampleEmbeddedTest, TestDocumentObjectBinder, HttpSolrClientBuilderTest, ExclusiveOrEvaluatorTest, LessThanEqualToEvaluatorTest, OperationsTest, SineEvaluatorTest, CoalesceEvaluatorTest, NormalizeEvaluatorTest, ArcCosineEvaluatorTest, MultiplyEvaluatorTest, TestCloudSolrClientConnections, TestCollectionStateWatchers]
[junit4] Completed [51/145 (1!)] on J1 in 149.57s, 9 tests, 1 failure <<< FAILURES!
[...truncated 46328 lines...]