You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2017/10/28 13:03:21 UTC
[JENKINS] Lucene-Solr-Tests-7.x - Build # 196 - Still unstable
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/196/
1 tests failed.
FAILED: org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader
Error Message:
Could not load collection from ZK: forceleader_test_collection
Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: forceleader_test_collection
at __randomizedtesting.SeedInfo.seed([8BC74AD0A95B1D43:6D507E1090D9E422]:0)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
at org.apache.solr.common.cloud.ZkStateReader.getLeader(ZkStateReader.java:787)
at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:809)
at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:547)
at org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader(ForceLeaderTest.java:143)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/forceleader_test_collection/state.json
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
... 47 more
Build Log:
[...truncated 13453 lines...]
[junit4] Suite: org.apache.solr.cloud.ForceLeaderTest
[junit4] 2> 5056756 INFO (SUITE-ForceLeaderTest-seed#[8BC74AD0A95B1D43]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/init-core-data-001
[junit4] 2> 5056757 WARN (SUITE-ForceLeaderTest-seed#[8BC74AD0A95B1D43]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=14 numCloses=14
[junit4] 2> 5056757 INFO (SUITE-ForceLeaderTest-seed#[8BC74AD0A95B1D43]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 5056791 INFO (SUITE-ForceLeaderTest-seed#[8BC74AD0A95B1D43]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 5056791 INFO (SUITE-ForceLeaderTest-seed#[8BC74AD0A95B1D43]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /_/e
[junit4] 2> 5056848 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 5056848 INFO (Thread-2793) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 5056848 INFO (Thread-2793) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 5056883 ERROR (Thread-2793) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 5056956 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkTestServer start zk server on port:39184
[junit4] 2> 5057031 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 5057032 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 5057045 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 5057046 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 5057047 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 5057047 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 5057048 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 5057048 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 5057049 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 5057083 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 5057084 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 5057103 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 5058141 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 5058284 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@63404405{/_/e,null,AVAILABLE}
[junit4] 2> 5058284 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@705906fa{HTTP/1.1,[http/1.1]}{127.0.0.1:37146}
[junit4] 2> 5058284 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server Started @5075560ms
[junit4] 2> 5058285 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/tempDir-001/control/data, replicaType=NRT, hostContext=/_/e, hostPort=43207, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/control-001/cores}
[junit4] 2> 5058285 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5058285 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 5058285 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5058285 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5058285 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-28T10:57:06.126Z
[junit4] 2> 5058305 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5058305 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/control-001/solr.xml
[junit4] 2> 5058378 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 5058378 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 5058379 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5058518 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39184/solr
[junit4] 2> 5059052 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5059065 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43207__%2Fe
[junit4] 2> 5059066 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.Overseer Overseer (id=98906159494201348-127.0.0.1:43207__%2Fe-n_0000000000) starting
[junit4] 2> 5059140 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43207__%2Fe
[junit4] 2> 5059175 INFO (zkCallback-1780-thread-1-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5059726 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5059872 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5059873 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5059907 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/control-001/cores
[junit4] 2> 5060110 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5060111 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39184/solr ready
[junit4] 2> 5060111 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=45701,localport=43207], receiveBufferSize:531000
[junit4] 2> 5060111 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37146,localport=58176], receiveBufferSize=530904
[junit4] 2> 5060165 INFO (qtp1430705358-12240) [n:127.0.0.1:43207__%2Fe ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:43207__%252Fe&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5060210 INFO (OverseerThreadFactory-5958-thread-1-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 5060353 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=45704,localport=43207], receiveBufferSize:531000
[junit4] 2> 5060364 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37146,localport=58180], receiveBufferSize=530904
[junit4] 2> 5060373 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 5060373 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 5060697 INFO (zkCallback-1780-thread-1-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 5061673 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 5061766 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 5062892 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 5063355 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 5063356 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5063356 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5063356 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 5063407 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39538988765470406]
[junit4] 2> 5063429 WARN (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 5064014 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5064014 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5064015 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5064015 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5064016 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1723384108, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 5064065 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4817dc57[control_collection_shard1_replica_n1] main]
[junit4] 2> 5064066 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 5064067 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 5064067 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5064068 INFO (searcherExecutor-5961-thread-1-processing-n:127.0.0.1:43207__%2Fe x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@4817dc57[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5064069 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582498559463260160
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:43207/_/e/control_collection_shard1_replica_n1/
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:43207/_/e/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 5064126 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 5064140 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:43207/_/e/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 5064224 INFO (zkCallback-1780-thread-2-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 5064248 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 5064249 INFO (qtp1430705358-12238) [n:127.0.0.1:43207__%2Fe c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3876
[junit4] 2> 5064306 INFO (qtp1430705358-12240) [n:127.0.0.1:43207__%2Fe ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 5064448 INFO (zkCallback-1780-thread-1-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 5065306 INFO (qtp1430705358-12240) [n:127.0.0.1:43207__%2Fe ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:43207__%252Fe&wt=javabin&version=2} status=0 QTime=5141
[junit4] 2> 5065490 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5065491 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39184/solr ready
[junit4] 2> 5065491 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 5065491 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=45744,localport=43207], receiveBufferSize:531000
[junit4] 2> 5065492 INFO (qtp1430705358-12241) [n:127.0.0.1:43207__%2Fe ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5065513 INFO (SocketProxy-Acceptor-43207) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37146,localport=58220], receiveBufferSize=530904
[junit4] 2> 5065526 INFO (OverseerCollectionConfigSetProcessor-98906159494201348-127.0.0.1:43207__%2Fe-n_0000000000) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5065526 INFO (OverseerThreadFactory-5958-thread-2-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 5065527 WARN (OverseerThreadFactory-5958-thread-2-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 5065783 INFO (qtp1430705358-12241) [n:127.0.0.1:43207__%2Fe ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 5065783 INFO (qtp1430705358-12241) [n:127.0.0.1:43207__%2Fe ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=291
[junit4] 2> 5067243 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001 of type TLOG
[junit4] 2> 5067243 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 5067396 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@25b7b6f9{/_/e,null,AVAILABLE}
[junit4] 2> 5067396 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@109cddf1{HTTP/1.1,[http/1.1]}{127.0.0.1:36794}
[junit4] 2> 5067396 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server Started @5084672ms
[junit4] 2> 5067396 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/_/e, hostPort=42162, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001/cores}
[junit4] 2> 5067396 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5067396 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 5067397 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5067397 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5067397 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-28T10:57:15.238Z
[junit4] 2> 5067543 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5067544 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001/solr.xml
[junit4] 2> 5067544 INFO (OverseerCollectionConfigSetProcessor-98906159494201348-127.0.0.1:43207__%2Fe-n_0000000000) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5067599 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 5067599 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 5067615 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5067637 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39184/solr
[junit4] 2> 5067765 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 5067766 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5067800 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42162__%2Fe
[junit4] 2> 5067801 INFO (zkCallback-1780-thread-2-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5067920 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5067930 INFO (zkCallback-1788-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 5069101 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5069192 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5069192 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5069226 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001/cores
[junit4] 2> 5069393 INFO (SocketProxy-Acceptor-42162) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=41154,localport=42162], receiveBufferSize:531000
[junit4] 2> 5069461 INFO (SocketProxy-Acceptor-42162) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=36794,localport=34552], receiveBufferSize=530904
[junit4] 2> 5069464 INFO (qtp1028606452-12294) [n:127.0.0.1:42162__%2Fe ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:42162__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5069520 INFO (OverseerThreadFactory-5958-thread-3-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:42162__%2Fe for creating new replica
[junit4] 2> 5069539 INFO (SocketProxy-Acceptor-42162) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=41158,localport=42162], receiveBufferSize:531000
[junit4] 2> 5069588 INFO (SocketProxy-Acceptor-42162) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=36794,localport=34556], receiveBufferSize=530904
[junit4] 2> 5069588 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 5069588 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 5069809 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 5069919 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema [collection1_shard2_replica_t41] Schema name=test
[junit4] 2> 5070974 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 5071066 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t41' using configuration from collection collection1, trusted=true
[junit4] 2> 5071067 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t41' (registry 'solr.core.collection1.shard2.replica_t41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5071067 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5071067 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [[collection1_shard2_replica_t41] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001/cores/collection1_shard2_replica_t41], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-1-001/cores/collection1_shard2_replica_t41/data/]
[junit4] 2> 5071102 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39538988765470406]
[junit4] 2> 5071175 WARN (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 5071539 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5071539 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5071541 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5071541 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5071561 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1723384108, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 5071561 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.SolrIndexSearcher Opening [Searcher@74c1f867[collection1_shard2_replica_t41] main]
[junit4] 2> 5071576 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 5071577 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 5071577 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5071578 INFO (searcherExecutor-5972-thread-1-processing-n:127.0.0.1:42162__%2Fe x:collection1_shard2_replica_t41 s:shard2 c:collection1) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [collection1_shard2_replica_t41] Registered new searcher Searcher@74c1f867[collection1_shard2_replica_t41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5071578 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582498567337017344
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:42162/_/e/collection1_shard2_replica_t41/
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy http://127.0.0.1:42162/_/e/collection1_shard2_replica_t41/ has no replicas
[junit4] 2> 5071652 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 5071665 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController collection1_shard2_replica_t41 stopping background replication from leader
[junit4] 2> 5071685 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5071685 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:42162/_/e/collection1_shard2_replica_t41/ shard2
[junit4] 2> 5071686 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 5071686 INFO (qtp1028606452-12292) [n:127.0.0.1:42162__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2098
[junit4] 2> 5071688 INFO (qtp1028606452-12294) [n:127.0.0.1:42162__%2Fe ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:42162__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=2223
[junit4] 2> 5071958 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 5073318 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001 of type TLOG
[junit4] 2> 5073319 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 5073427 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7c2fdca1{/_/e,null,AVAILABLE}
[junit4] 2> 5073427 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5543f653{HTTP/1.1,[http/1.1]}{127.0.0.1:33348}
[junit4] 2> 5073427 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server Started @5090703ms
[junit4] 2> 5073427 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/_/e, hostPort=41891, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001/cores}
[junit4] 2> 5073428 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5073465 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 5073465 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5073465 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5073465 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-28T10:57:21.306Z
[junit4] 2> 5073502 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5073503 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001/solr.xml
[junit4] 2> 5073516 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f62a0c6c0000a, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 5073518 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 5073518 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 5073520 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5073553 INFO (OverseerCollectionConfigSetProcessor-98906159494201348-127.0.0.1:43207__%2Fe-n_0000000000) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5073569 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39184/solr
[junit4] 2> 5073755 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 5073772 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5073774 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41891__%2Fe
[junit4] 2> 5073774 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5073886 INFO (zkCallback-1788-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5073959 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5073882 INFO (zkCallback-1780-thread-2-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 5075013 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5075124 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5075125 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5075126 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001/cores
[junit4] 2> 5075526 INFO (SocketProxy-Acceptor-41891) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=38412,localport=41891], receiveBufferSize:531000
[junit4] 2> 5075582 INFO (SocketProxy-Acceptor-41891) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=33348,localport=47020], receiveBufferSize=530904
[junit4] 2> 5075603 INFO (qtp994154853-12329) [n:127.0.0.1:41891__%2Fe ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41891__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5075638 INFO (OverseerThreadFactory-5958-thread-4-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:41891__%2Fe for creating new replica
[junit4] 2> 5075692 INFO (SocketProxy-Acceptor-41891) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=38416,localport=41891], receiveBufferSize:531000
[junit4] 2> 5075693 INFO (SocketProxy-Acceptor-41891) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=33348,localport=47024], receiveBufferSize=530904
[junit4] 2> 5075729 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 5075730 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 5075746 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 5076118 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 5076209 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema [collection1_shard1_replica_t43] Schema name=test
[junit4] 2> 5077525 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 5077675 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t43' using configuration from collection collection1, trusted=true
[junit4] 2> 5077676 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t43' (registry 'solr.core.collection1.shard1.replica_t43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5077676 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5077676 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [[collection1_shard1_replica_t43] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001/cores/collection1_shard1_replica_t43], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-2-001/cores/collection1_shard1_replica_t43/data/]
[junit4] 2> 5077691 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39538988765470406]
[junit4] 2> 5077729 WARN (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 5078419 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5078419 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5078420 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5078420 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5078421 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1723384108, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 5078422 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.SolrIndexSearcher Opening [Searcher@54678c50[collection1_shard1_replica_t43] main]
[junit4] 2> 5078439 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 5078439 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 5078440 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5078440 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582498574532345856
[junit4] 2> 5078442 INFO (searcherExecutor-5983-thread-1-processing-n:127.0.0.1:41891__%2Fe x:collection1_shard1_replica_t43 s:shard1 c:collection1) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [collection1_shard1_replica_t43] Registered new searcher Searcher@54678c50[collection1_shard1_replica_t43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:41891/_/e/collection1_shard1_replica_t43/
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy http://127.0.0.1:41891/_/e/collection1_shard1_replica_t43/ has no replicas
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 5078457 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController collection1_shard1_replica_t43 stopping background replication from leader
[junit4] 2> 5078471 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:41891/_/e/collection1_shard1_replica_t43/ shard1
[junit4] 2> 5078493 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 5078499 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 5078528 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 5078530 INFO (qtp994154853-12327) [n:127.0.0.1:41891__%2Fe c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2801
[junit4] 2> 5078593 INFO (qtp994154853-12329) [n:127.0.0.1:41891__%2Fe ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:41891__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=2990
[junit4] 2> 5078639 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 5078646 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 5079651 INFO (OverseerCollectionConfigSetProcessor-98906159494201348-127.0.0.1:43207__%2Fe-n_0000000000) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 5080691 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001 of type TLOG
[junit4] 2> 5080708 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 5080952 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4a46eab2{/_/e,null,AVAILABLE}
[junit4] 2> 5080953 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6b991fb7{HTTP/1.1,[http/1.1]}{127.0.0.1:39542}
[junit4] 2> 5080953 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.e.j.s.Server Started @5098228ms
[junit4] 2> 5080953 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/tempDir-001/jetty3, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/_/e, hostPort=41221, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001/cores}
[junit4] 2> 5080953 ERROR (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 5080962 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 5080962 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 5080962 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 5080963 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-28T10:57:28.804Z
[junit4] 2> 5081054 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 5081055 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001/solr.xml
[junit4] 2> 5081074 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 5081074 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 5081145 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 5081165 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39184/solr
[junit4] 2> 5081367 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 5081384 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 5081385 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41221__%2Fe
[junit4] 2> 5081400 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5081496 INFO (zkCallback-1805-thread-1-processing-n:127.0.0.1:41221__%2Fe) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5081461 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5081461 INFO (zkCallback-1780-thread-1-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5081461 INFO (zkCallback-1788-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 5081951 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5082100 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5082133 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5082135 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001/cores
[junit4] 2> 5082537 INFO (SocketProxy-Acceptor-41221) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=53178,localport=41221], receiveBufferSize:531000
[junit4] 2> 5082556 INFO (SocketProxy-Acceptor-41221) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=39542,localport=40134], receiveBufferSize=530904
[junit4] 2> 5082596 INFO (qtp1223044078-12363) [n:127.0.0.1:41221__%2Fe ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41221__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 5082704 INFO (OverseerThreadFactory-5958-thread-5-processing-n:127.0.0.1:43207__%2Fe) [n:127.0.0.1:43207__%2Fe ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:41221__%2Fe for creating new replica
[junit4] 2> 5082705 INFO (SocketProxy-Acceptor-41221) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=53182,localport=41221], receiveBufferSize:531000
[junit4] 2> 5082772 INFO (SocketProxy-Acceptor-41221) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=39542,localport=40138], receiveBufferSize=530904
[junit4] 2> 5082772 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 5082773 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 5082794 INFO (zkCallback-1805-thread-1-processing-n:127.0.0.1:41221__%2Fe) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5082794 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5082794 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5083918 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 5084193 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.s.IndexSchema [collection1_shard2_replica_t45] Schema name=test
[junit4] 2> 5085348 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 5085438 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t45' using configuration from collection collection1, trusted=true
[junit4] 2> 5085439 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t45' (registry 'solr.core.collection1.shard2.replica_t45') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d0fd356
[junit4] 2> 5085439 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 5085439 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.SolrCore [[collection1_shard2_replica_t45] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001/cores/collection1_shard2_replica_t45], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001/shard-3-001/cores/collection1_shard2_replica_t45/data/]
[junit4] 2> 5085494 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39538988765470406]
[junit4] 2> 5085511 WARN (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 5086095 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 5086095 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 5086096 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 5086096 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 5086097 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1723384108, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 5086098 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.s.SolrIndexSearcher Opening [Searcher@234ce6ec[collection1_shard2_replica_t45] main]
[junit4] 2> 5086132 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 5086132 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 5086133 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 5086134 INFO (searcherExecutor-5994-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.SolrCore [collection1_shard2_replica_t45] Registered new searcher Searcher@234ce6ec[collection1_shard2_replica_t45] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 5086134 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582498582600089600
[junit4] 2> 5086186 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.ZkController Core needs to recover:collection1_shard2_replica_t45
[junit4] 2> 5086187 INFO (qtp1223044078-12365) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=3414
[junit4] 2> 5086188 INFO (updateExecutor-1802-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 5086201 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 5086278 INFO (qtp1223044078-12363) [n:127.0.0.1:41221__%2Fe ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:41221__%252Fe&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=3682
[junit4] 2> 5086331 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 5086331 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.ZkController collection1_shard2_replica_t45 stopping background replication from leader
[junit4] 2> 5086331 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard2_replica_t45]
[junit4] 2> 5086331 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 5086331 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard2_replica_t45] as recovering, leader is [http://127.0.0.1:42162/_/e/collection1_shard2_replica_t41/] and I am [http://127.0.0.1:41221/_/e/collection1_shard2_replica_t45/]
[junit4] 2> 5086332 INFO (zkCallback-1805-thread-1-processing-n:127.0.0.1:41221__%2Fe) [n:127.0.0.1:41221__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5086335 INFO (zkCallback-1793-thread-1-processing-n:127.0.0.1:42162__%2Fe) [n:127.0.0.1:42162__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5086335 INFO (zkCallback-1799-thread-1-processing-n:127.0.0.1:41891__%2Fe) [n:127.0.0.1:41891__%2Fe ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 5086344 INFO (TEST-ForceLeaderTest.testLastPublishedStateIsActive-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testLastPublishedStateIsActive
[junit4] 2> 5086345 INFO (recoveryExecutor-1803-thread-1-processing-n:127.0.0.1:41221__%2Fe x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:41221__%2Fe c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:4216
[...truncated too long message...]
> 5212092 INFO (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40794 40794
[junit4] 2> 5212367 INFO (Thread-2834) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40794 40794
[junit4] 2> 5212387 WARN (Thread-2834) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/aliases.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 4 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/clusterprops.json
[junit4] 2> 7 /solr/clusterstate.json
[junit4] 2> 3 /solr/collections/forceleader_test_collection/state.json
[junit4] 2> 3 /solr/collections/collection1/state.json
[junit4] 2> 3 /solr/autoscaling.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/live_nodes
[junit4] 2> 7 /solr/collections
[junit4] 2> 3 /solr/overseer/collection-queue-work
[junit4] 2> 3 /solr/overseer/queue-work
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2>
[junit4] 2> 5212387 WARN (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SocketProxy Closing 16 connections to: http://127.0.0.1:41867/_/e, target: http://127.0.0.1:34584/_/e
[junit4] 2> 5212400 WARN (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SocketProxy Closing 0 connections to: http://127.0.0.1:36363/_/e, target: http://127.0.0.1:43221/_/e
[junit4] 2> 5212400 WARN (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SocketProxy Closing 4 connections to: http://127.0.0.1:43097/_/e, target: http://127.0.0.1:40800/_/e
[junit4] 2> 5212400 WARN (TEST-ForceLeaderTest.testReplicasInLIRNoLeader-seed#[8BC74AD0A95B1D43]) [ ] o.a.s.c.SocketProxy Closing 7 connections to: http://127.0.0.1:41330/_/e, target: http://127.0.0.1:43778/_/e
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ForceLeaderTest -Dtests.method=testReplicasInLIRNoLeader -Dtests.seed=8BC74AD0A95B1D43 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=tr-TR -Dtests.timezone=Africa/Ouagadougou -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 64.6s J1 | ForceLeaderTest.testReplicasInLIRNoLeader <<<
[junit4] > Throwable #1: org.apache.solr.common.SolrException: Could not load collection from ZK: forceleader_test_collection
[junit4] > at __randomizedtesting.SeedInfo.seed([8BC74AD0A95B1D43:6D507E1090D9E422]:0)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1172)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:692)
[junit4] > at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.getLeader(ZkStateReader.java:787)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:809)
[junit4] > at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:547)
[junit4] > at org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader(ForceLeaderTest.java:143)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/forceleader_test_collection/state.json
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] > at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1184)
[junit4] > at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1170)
[junit4] > ... 47 more
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_8BC74AD0A95B1D43-001
[junit4] 2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=7901, maxDocsPerChunk=2, blockSize=5), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=7901, blockSize=5)), sim=RandomSimilarity(queryNorm=true): {}, locale=tr-TR, timezone=Africa/Ouagadougou
[junit4] 2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=274698120,total=522715136
[junit4] 2> NOTE: All tests run in this JVM: [OverseerCollectionConfigSetProcessorTest, TestLazyCores, TestCloudNestedDocsSort, DocValuesMultiTest, DistributedTermsComponentTest, TestOverriddenPrefixQueryForCustomFieldType, TestMaxTokenLenTokenizer, TestExactStatsCache, OpenExchangeRatesOrgProviderTest, HdfsUnloadDistributedZkTest, SolrJmxReporterTest, TestComplexPhraseQParserPlugin, SolrGangliaReporterTest, DirectUpdateHandlerTest, TestZkAclsWithHadoopAuth, CdcrBootstrapTest, TestDefaultStatsCache, DistributedDebugComponentTest, TestSolr4Spatial, TestConfigSetsAPIZkFailure, DistributedFacetPivotLargeTest, TestCloudPivotFacet, TestRTGBase, TestDynamicFieldResource, ShardRoutingTest, CoreAdminRequestStatusTest, TestPseudoReturnFields, TestSolrQueryParser, TestSolrCoreSnapshots, SolrRequestParserTest, TestMacros, HighlighterConfigTest, MBeansHandlerTest, TestScoreJoinQPScore, CdcrRequestHandlerTest, SolrIndexSplitterTest, ConfigureRecoveryStrategyTest, TestManagedResource, TestBackupRepositoryFactory, NodeAddedTriggerTest, OverseerStatusTest, SimpleMLTQParserTest, V2StandaloneTest, TestMiniSolrCloudClusterSSL, TestDistributedStatsComponentCardinality, TestFieldTypeCollectionResource, TestAtomicUpdateErrorCases, PreAnalyzedFieldTest, TestDocumentBuilder, TestFastWriter, AssignTest, TestSegmentSorting, HighlighterMaxOffsetTest, BlobRepositoryCloudTest, TestSerializedLuceneMatchVersion, TestAnalyzedSuggestions, LukeRequestHandlerTest, TestPushWriter, TestManagedSchema, ShowFileRequestHandlerTest, TestSkipOverseerOperations, TestSearcherReuse, TestLegacyTerms, DeleteLastCustomShardedReplicaTest, DateRangeFieldTest, TestFieldCacheWithThreads, HttpPartitionTest, ShardRoutingCustomTest, TestInfoStreamLogging, TestCustomStream, TestRequestForwarding, TestBulkSchemaAPI, SolrIndexConfigTest, ConcurrentDeleteAndCreateCollectionTest, SampleTest, TestCollectionAPIs, HdfsSyncSliceTest, TestPointFields, TestCollationField, TestRangeQuery, TestPivotHelperCode, TestFilteredDocIdSet, SuggesterTSTTest, TestFaceting, TestHighFrequencyDictionaryFactory, TestBinaryResponseWriter, DistribDocExpirationUpdateProcessorTest, UninvertDocValuesMergePolicyTest, TestNumericRangeQuery64, TemplateUpdateProcessorTest, TestReRankQParserPlugin, SpatialRPTFieldTypeTest, TestSortByMinMaxFunction, TestStressRecovery, HLLSerializationTest, TestDocSet, CircularListTest, BlockJoinFacetRandomTest, TestStressReorder, TestDocTermOrds, TestWriterPerf, SpatialHeatmapFacetsTest, ConvertedLegacyTest, ConjunctionSolrSpellCheckerTest, SOLR749Test, TestExceedMaxTermLength, DistributedFacetPivotSmallTest, OpenCloseCoreStressTest, DeleteShardTest, TestQueryWrapperFilter, SharedFSAutoReplicaFailoverTest, HdfsBasicDistributedZk2Test, SSLMigrationTest, TestFieldCacheSort, TestRetrieveFieldsOptimizer, TestBM25SimilarityFactory, TestJettySolrRunner, TestUseDocValuesAsStored, DistributedQueryComponentOptimizationTest, AddReplicaTest, TestXmlQParser, SynonymTokenizerTest, TestDistributedGrouping, TestHashPartitioner, TestRealTimeGet, TestCoreContainer, TestGroupingSearch, PeerSyncTest, TestSort, BasicFunctionalityTest, DistributedQueryElevationComponentTest, CoreAdminHandlerTest, TestCSVLoader, NoCacheHeaderTest, SolrCoreCheckLockOnStartupTest, QueryEqualityTest, TestRemoteStreaming, TestSolrDeletionPolicy1, SolrInfoBeanTest, IndexBasedSpellCheckerTest, DisMaxRequestHandlerTest, DocumentAnalysisRequestHandlerTest, RequestHandlersTest, PathHierarchyTokenizerFactoryTest, FieldAnalysisRequestHandlerTest, LoggingHandlerTest, IndexSchemaTest, QueryParsingTest, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, AlternateDirectoryTest, TestSolrIndexConfig, CopyFieldTest, TestSolrDeletionPolicy2, NumericFieldsTest, OutputWriterTest, DirectSolrConnectionTest, TestPhraseSuggestions, TestPerFieldSimilarity, PrimUtilsTest, SpellingQueryConverterTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestUtils, SystemInfoHandlerTest, DistributedIntervalFacetingTest, TestSimpleTrackingShardHandler, BaseCdcrDistributedZkTest, CdcrReplicationDistributedZkTest, CdcrReplicationHandlerTest, ChaosMonkeySafeLeaderWithPullReplicasTest, DeleteInactiveReplicaTest, DistribJoinFromCollectionTest, ForceLeaderTest]
[junit4] Completed [531/745 (1!)] on J1 in 156.01s, 3 tests, 1 error, 1 skipped <<< FAILURES!
[...truncated 47937 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 202 - Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/202/
All tests passed
Build Log:
[...truncated 4708 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:826: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:770: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build.xml:495: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:2262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:409: impossible to resolve dependencies:
resolve failed - see output for details
Total time: 115 minutes 8 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 201 - Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/201/
All tests passed
Build Log:
[...truncated 4723 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:826: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:770: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build.xml:495: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:2262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:409: impossible to resolve dependencies:
resolve failed - see output for details
Total time: 117 minutes 26 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 200 - Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/200/
All tests passed
Build Log:
[...truncated 1364 lines...]
[junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5418177546657407218320.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] #
[junit4] # There is insufficient memory for the Java Runtime Environment to continue.
[junit4] # Native memory allocation (mmap) failed to map 41418752 bytes for committing reserved memory.
[junit4] # An error report file with more information is saved as:
[junit4] # /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/J1/hs_err_pid10311.log
[junit4] <<< JVM J1: EOF ----
[junit4] JVM J1: stderr was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5411283484380853938762.syserr
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000000ed080000, 41418752, 0) failed; error='Cannot allocate memory' (errno=12)
[junit4] <<< JVM J1: EOF ----
[...truncated 288 lines...]
[junit4] ERROR: JVM J1 ended with an exception, command line: /usr/local/asfpackages/java/jdk1.8.0_144/jre/bin/java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=AB360698341A6835 -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=europarl.lines.txt.gz -Dtests.luceneMatchVersion=7.2.0 -Dtests.cleanthreads=perMethod -Djava.util.logging.config.file=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/tools/junit4/logging.properties -Dtests.nightly=false -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=2 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp -Dcommon.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene -Dclover.db.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/clover/db -Djava.security.policy=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/tools/junit4/tests.policy -Dtests.LUCENE_VERSION=7.2.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Dtests.src.home=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x -Djava.security.egd=file:/dev/./urandom -Djunit4.childvm.cwd=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/J1 -Djunit4.childvm.id=1 -Djunit4.childvm.count=3 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -Dfile.encoding=ISO-8859-1 -classpath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/codecs/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/test-framework/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/test-framework/lib/junit-4.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/test-framework/lib/randomizedtesting-runner-2.5.3.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/classes/test:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5418151426299979181834.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5412074331131381671030.suites -stdin
[junit4] ERROR: JVM J1 ended with an exception: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5418177546657407218320.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5411283484380853938762.syserr
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] at java.lang.Thread.run(Thread.java:748)
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:826: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:770: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build.xml:50: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1494: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1021: At least one slave process threw an exception, first: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5418177546657407218320.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/core/test/temp/junit4-J1-20171031_141921_5411283484380853938762.syserr
Total time: 46 minutes 59 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-Tests-7.x #197 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 199 - Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/199/
1 tests failed.
FAILED: org.apache.solr.cloud.TestLocalFSCloudBackupRestore.test
Error Message:
expected:<COMPLETED> but was:<RUNNING>
Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
at __randomizedtesting.SeedInfo.seed([39042BD03F00148:8BC47D67AD0C6CB0]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:277)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11578 lines...]
[junit4] Suite: org.apache.solr.cloud.TestLocalFSCloudBackupRestore
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/init-core-data-001
[junit4] 2> 0 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 704 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.e.j.u.log Logging initialized @26167ms
[junit4] 2> 801 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 886 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 1035 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001
[junit4] 2> 1153 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1156 INFO (Thread-1) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1156 INFO (Thread-1) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1884 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:35952
[junit4] 2> 2990 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
[junit4] 2> 3627 ERROR (Thread-1) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 5182 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f7083c0440000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 9513 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 9549 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 9930 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@34e710a4{/solr,null,AVAILABLE}
[junit4] 2> 9964 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7db77681{/solr,null,AVAILABLE}
[junit4] 2> 10051 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@39e9b53{HTTP/1.1,[http/1.1]}{127.0.0.1:41201}
[junit4] 2> 10052 INFO (jetty-launcher-1-thread-2) [ ] o.e.j.s.Server Started @35612ms
[junit4] 2> 10052 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41201}
[junit4] 2> 10055 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5a6ff595{HTTP/1.1,[http/1.1]}{127.0.0.1:58560}
[junit4] 2> 10056 INFO (jetty-launcher-1-thread-1) [ ] o.e.j.s.Server Started @35616ms
[junit4] 2> 10056 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58560}
[junit4] 2> 10371 ERROR (jetty-launcher-1-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 10371 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 10375 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 10375 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 10376 ERROR (jetty-launcher-1-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 10377 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 10428 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 10428 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 10487 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-31T03:40:11.947Z
[junit4] 2> 10551 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-31T03:40:11.911Z
[junit4] 2> 10667 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
[junit4] 2> 10972 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 11026 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 18546 INFO (jetty-launcher-1-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35952/solr
[junit4] 2> 18551 INFO (jetty-launcher-1-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35952/solr
[junit4] 2> 18680 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f7083c0440004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 20262 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 20270 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:41201_solr
[junit4] 2> 20280 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 20303 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.c.Overseer Overseer (id=98921427994148869-127.0.0.1:41201_solr-n_0000000000) starting
[junit4] 2> 20307 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58560_solr
[junit4] 2> 20378 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 20379 INFO (zkCallback-12-thread-1-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 21700 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41201_solr
[junit4] 2> 21757 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 21763 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 27189 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27221 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27222 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27466 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27500 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27500 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 27692 INFO (jetty-launcher-1-thread-2) [n:127.0.0.1:41201_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/.
[junit4] 2> 27764 INFO (jetty-launcher-1-thread-1) [n:127.0.0.1:58560_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/.
[junit4] 2> 29970 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 29972 INFO (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35952/solr ready
[junit4] 2> 30097 INFO (TEST-TestLocalFSCloudBackupRestore.test-seed#[39042BD03F00148]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 33822 INFO (qtp1422851308-35) [n:127.0.0.1:58560_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&router.name=implicit&version=2&pullReplicas=1&shards=shard1,shard2&property.customKey=customValue&maxShardsPerNode=3&router.field=shard_s&autoAddReplicas=true&name=backuprestore&nrtReplicas=1&action=CREATE&tlogReplicas=1&wt=javabin and sendToOCPQueue=true
[junit4] 2> 34064 INFO (OverseerThreadFactory-12-thread-1-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.CreateCollectionCmd Create collection backuprestore
[junit4] 2> 34070 WARN (OverseerThreadFactory-12-thread-1-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 3 on collection backuprestore is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
[junit4] 2> 34296 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"backuprestore_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:58560/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 34534 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"backuprestore_shard1_replica_t2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:41201/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 34536 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"backuprestore_shard1_replica_p3",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:58560/solr",
[junit4] 2> "type":"PULL",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 34552 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"backuprestore_shard2_replica_n4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:41201/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 34553 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"backuprestore_shard2_replica_t5",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:58560/solr",
[junit4] 2> "type":"TLOG",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 34572 INFO (OverseerStateUpdate-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"backuprestore",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"backuprestore_shard2_replica_p6",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:41201/solr",
[junit4] 2> "type":"PULL",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 35287 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node11&name=backuprestore_shard2_replica_t5&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 35287 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node10&name=backuprestore_shard2_replica_n4&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 35325 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node9&name=backuprestore_shard1_replica_p3&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 35326 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node7&name=backuprestore_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 35329 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node8&name=backuprestore_shard1_replica_t2&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 35388 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 35390 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 35416 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node12&name=backuprestore_shard2_replica_p6&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 35721 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 35727 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 36801 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 36843 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 38736 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 38756 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 38770 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.IndexSchema [backuprestore_shard2_replica_p6] Schema name=minimal
[junit4] 2> 38776 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 38820 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 38848 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.IndexSchema [backuprestore_shard1_replica_t2] Schema name=minimal
[junit4] 2> 40270 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.IndexSchema [backuprestore_shard1_replica_p3] Schema name=minimal
[junit4] 2> 40279 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.s.IndexSchema [backuprestore_shard2_replica_t5] Schema name=minimal
[junit4] 2> 40284 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.s.IndexSchema [backuprestore_shard2_replica_n4] Schema name=minimal
[junit4] 2> 40337 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.s.IndexSchema [backuprestore_shard1_replica_n1] Schema name=minimal
[junit4] 2> 40625 WARN (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40630 WARN (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieLongField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40655 WARN (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40659 WARN (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40670 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 40671 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_n4' using configuration from collection backuprestore, trusted=true
[junit4] 2> 40671 WARN (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40679 WARN (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40680 WARN (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SolrResourceLoader Solr loaded a deprecated plugin/analysis class [solr.TrieIntField]. Please consult documentation how to replace it accordingly.
[junit4] 2> 40782 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 40782 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_p6' using configuration from collection backuprestore, trusted=true
[junit4] 2> 41017 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 41022 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 41022 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 41017 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_t2' using configuration from collection backuprestore, trusted=true
[junit4] 2> 41028 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 41028 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica_t5' using configuration from collection backuprestore, trusted=true
[junit4] 2> 41022 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_n1' using configuration from collection backuprestore, trusted=true
[junit4] 2> 41022 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica_p3' using configuration from collection backuprestore, trusted=true
[junit4] 2> 41228 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.core.backuprestore.shard1.replica_t2' (registry 'solr.core.backuprestore.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41232 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.core.backuprestore.shard1.replica_p3' (registry 'solr.core.backuprestore.shard1.replica_p3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41232 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.core.backuprestore.shard1.replica_n1' (registry 'solr.core.backuprestore.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41232 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58560.solr.core.backuprestore.shard2.replica_t5' (registry 'solr.core.backuprestore.shard2.replica_t5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41233 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.core.backuprestore.shard2.replica_p6' (registry 'solr.core.backuprestore.shard2.replica_p6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41253 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41263 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41268 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41268 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41270 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrCore [[backuprestore_shard2_replica_p6] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/backuprestore_shard2_replica_p6], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/./backuprestore_shard2_replica_p6/data/]
[junit4] 2> 41284 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SolrCore [[backuprestore_shard2_replica_t5] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/backuprestore_shard2_replica_t5], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/./backuprestore_shard2_replica_t5/data/]
[junit4] 2> 41299 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrCore [[backuprestore_shard1_replica_p3] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/backuprestore_shard1_replica_p3], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/./backuprestore_shard1_replica_p3/data/]
[junit4] 2> 41363 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrCore [[backuprestore_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/backuprestore_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node1/./backuprestore_shard1_replica_n1/data/]
[junit4] 2> 41367 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41367 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore [[backuprestore_shard1_replica_t2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/backuprestore_shard1_replica_t2], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/./backuprestore_shard1_replica_t2/data/]
[junit4] 2> 41376 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41201.solr.core.backuprestore.shard2.replica_n4' (registry 'solr.core.backuprestore.shard2.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18a5e02b
[junit4] 2> 41376 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 41376 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrCore [[backuprestore_shard2_replica_n4] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/backuprestore_shard2_replica_n4], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestLocalFSCloudBackupRestore_39042BD03F00148-001/tempDir-001/node2/./backuprestore_shard2_replica_n4/data/]
[junit4] 2> 52558 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 52558 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 52668 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 52711 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 52863 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 52863 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 52876 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 53333 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 53392 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 53392 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 53394 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 53394 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 53423 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 53424 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 53437 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 53437 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 53439 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 53439 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 53469 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 53470 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 53513 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@4ec697d8[backuprestore_shard2_replica_n4] main]
[junit4] 2> 53532 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.s.SolrIndexSearcher Opening [Searcher@314e8040[backuprestore_shard2_replica_t5] main]
[junit4] 2> 53532 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@bc28726[backuprestore_shard1_replica_n1] main]
[junit4] 2> 53533 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@3347e5b6[backuprestore_shard1_replica_t2] main]
[junit4] 2> 53559 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.SolrIndexSearcher Opening [Searcher@372436bd[backuprestore_shard2_replica_p6] main]
[junit4] 2> 53624 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.SolrIndexSearcher Opening [Searcher@76bf53d[backuprestore_shard1_replica_p3] main]
[junit4] 2> 53665 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53667 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53708 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53715 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53715 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53731 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53740 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53740 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53755 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 53760 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53765 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53775 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 53841 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 53878 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 53897 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 53897 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 53848 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 53898 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 57876 WARN (SUITE-TestLocalFSCloudBackupRestore-seed#[39042BD03F00148]-worker-SendThread(127.0.0.1:35952)) [ ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 6712ms for sessionid 0x15f7083c0440008
[junit4] 2> 57877 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f7083c0440008, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 58083 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582742906708426752
[junit4] 2> 58264 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 58266 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 58284 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 59122 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.ZkController backuprestore_shard1_replica_p3 starting background replication from leader
[junit4] 2> 59125 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582742907801042944
[junit4] 2> 59125 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.ZkController backuprestore_shard2_replica_p6 starting background replication from leader
[junit4] 2> 59126 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582742907802091520
[junit4] 2> 59132 WARN (zkCallback-20-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5077fabf name: ZooKeeperConnection Watcher:127.0.0.1:35952/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 59132 WARN (zkCallback-20-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 59216 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 59293 INFO (searcherExecutor-20-thread-1-processing-n:127.0.0.1:58560_solr x:backuprestore_shard1_replica_n1 s:shard1 c:backuprestore r:core_node7) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SolrCore [backuprestore_shard1_replica_n1] Registered new searcher Searcher@bc28726[backuprestore_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59294 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 59294 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 59295 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:58560/solr/backuprestore_shard1_replica_n1/
[junit4] 2> 59421 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=backuprestore_shard1_replica_n1 url=http://127.0.0.1:58560/solr START replicas=[http://127.0.0.1:41201/solr/backuprestore_shard1_replica_t2/] nUpdates=100
[junit4] 2> 59770 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=backuprestore_shard1_replica_n1 url=http://127.0.0.1:58560/solr DONE. We have no versions. sync failed.
[junit4] 2> 59773 INFO (searcherExecutor-18-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard2_replica_p6 s:shard2 c:backuprestore r:core_node12) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrCore [backuprestore_shard2_replica_p6] Registered new searcher Searcher@372436bd[backuprestore_shard2_replica_p6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59777 INFO (searcherExecutor-21-thread-1-processing-n:127.0.0.1:58560_solr x:backuprestore_shard2_replica_t5 s:shard2 c:backuprestore r:core_node11) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SolrCore [backuprestore_shard2_replica_t5] Registered new searcher Searcher@314e8040[backuprestore_shard2_replica_t5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59779 INFO (searcherExecutor-22-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard1_replica_t2 s:shard1 c:backuprestore r:core_node8) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore [backuprestore_shard1_replica_t2] Registered new searcher Searcher@3347e5b6[backuprestore_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59780 INFO (searcherExecutor-17-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard2_replica_n4 s:shard2 c:backuprestore r:core_node10) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.SolrCore [backuprestore_shard2_replica_n4] Registered new searcher Searcher@4ec697d8[backuprestore_shard2_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59782 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582742908489957376
[junit4] 2> 59231 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 59804 INFO (searcherExecutor-19-thread-1-processing-n:127.0.0.1:58560_solr x:backuprestore_shard1_replica_p3 s:shard1 c:backuprestore r:core_node9) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrCore [backuprestore_shard1_replica_p3] Registered new searcher Searcher@76bf53d[backuprestore_shard1_replica_p3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 59845 INFO (qtp1468422210-28) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.S.Request [backuprestore_shard1_replica_t2] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=392
[junit4] 2> 59851 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=9999ms
[junit4] 2> 59882 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 59882 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 59883 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 59883 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 60015 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60016 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60417 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 60417 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 60417 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:58560/solr/backuprestore_shard2_replica_t5/
[junit4] 2> 60419 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.PeerSync PeerSync: core=backuprestore_shard2_replica_t5 url=http://127.0.0.1:58560/solr START replicas=[http://127.0.0.1:41201/solr/backuprestore_shard2_replica_n4/] nUpdates=100
[junit4] 2> 60531 INFO (qtp1468422210-31) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.c.S.Request [backuprestore_shard2_replica_n4] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 60531 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 60532 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 60532 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 60616 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:58560/solr/backuprestore_shard1_replica_n1/ shard1
[junit4] 2> 60717 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60717 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60731 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.u.PeerSync PeerSync: core=backuprestore_shard2_replica_t5 url=http://127.0.0.1:58560/solr DONE. We have no versions. sync failed.
[junit4] 2> 60731 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 60731 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 60732 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 60732 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 60736 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ZkController backuprestore_shard2_replica_t5 stopping background replication from leader
[junit4] 2> 60756 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:58560/solr/backuprestore_shard2_replica_t5/ shard2
[junit4] 2> 60757 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node7&name=backuprestore_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin} status=0 QTime=25431
[junit4] 2> 60759 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Replica core_node7 is leader but it's state is down, skipping replication
[junit4] 2> 60860 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60861 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 60863 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 60880 INFO (qtp1422851308-33) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node11&name=backuprestore_shard2_replica_t5&action=CREATE&numShards=2&shard=shard2&wt=javabin} status=0 QTime=25599
[junit4] 2> 60882 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Replica core_node11 is leader but it's state is down, skipping replication
[junit4] 2> 60903 INFO (qtp1468422210-25) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node10 x:backuprestore_shard2_replica_n4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=NRT&property.customKey=customValue&coreNodeName=core_node10&name=backuprestore_shard2_replica_n4&action=CREATE&numShards=2&shard=shard2&wt=javabin} status=0 QTime=25619
[junit4] 2> 60905 INFO (qtp1422851308-37) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node9&name=backuprestore_shard1_replica_p3&action=CREATE&numShards=2&shard=shard1&wt=javabin} status=0 QTime=25580
[junit4] 2> 60940 INFO (qtp1468422210-29) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=PULL&property.customKey=customValue&coreNodeName=core_node12&name=backuprestore_shard2_replica_p6&action=CREATE&numShards=2&shard=shard2&wt=javabin} status=0 QTime=25524
[junit4] 2> 61052 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 61053 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 61432 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.ZkController backuprestore_shard1_replica_t2 starting background replication from leader
[junit4] 2> 61432 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 61449 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 61449 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 61452 INFO (qtp1468422210-27) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=backuprestore&version=2&replicaType=TLOG&property.customKey=customValue&coreNodeName=core_node8&name=backuprestore_shard1_replica_t2&action=CREATE&numShards=2&shard=shard1&wt=javabin} status=0 QTime=26123
[junit4] 2> 61461 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:58560/solr/backuprestore_shard1_replica_n1/
[junit4] 2> 61615 INFO (zkCallback-13-thread-1-processing-n:127.0.0.1:58560_solr) [n:127.0.0.1:58560_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 61616 INFO (zkCallback-12-thread-2-processing-n:127.0.0.1:41201_solr) [n:127.0.0.1:41201_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
[junit4] 2> 61628 INFO (qtp1422851308-36) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.S.Request [backuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=150
[junit4] 2> 61636 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 61648 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 61648 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 61648 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 61649 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 61676 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@7af42f8f[backuprestore_shard1_replica_t2] main]
[junit4] 2> 61705 INFO (searcherExecutor-22-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard1_replica_t2 s:shard1 c:backuprestore r:core_node8) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore [backuprestore_shard1_replica_t2] Registered new searcher Searcher@7af42f8f[backuprestore_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 61965 INFO (OverseerCollectionConfigSetProcessor-98921427994148869-127.0.0.1:41201_solr-n_0000000000) [n:127.0.0.1:41201_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 62660 INFO (qtp1422851308-35) [n:127.0.0.1:58560_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 62661 INFO (qtp1422851308-35) [n:127.0.0.1:58560_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&router.name=implicit&version=2&pullReplicas=1&shards=shard1,shard2&property.customKey=customValue&maxShardsPerNode=3&router.field=shard_s&autoAddReplicas=true&name=backuprestore&nrtReplicas=1&action=CREATE&tlogReplicas=1&wt=javabin} status=0 QTime=29127
[junit4] 2> 62883 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication
[junit4] 2> 62884 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:58560/solr/backuprestore_shard1_replica_n1/
[junit4] 2> 62885 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Last replication failed, so I'll force replication
[junit4] 2> 62885 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:58560/solr/backuprestore_shard2_replica_t5/
[junit4] 2> 62886 INFO (qtp1422851308-39) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.S.Request [backuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 62892 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 62892 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 62892 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 62892 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 62892 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 62896 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 62896 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2c8c1df2 commitCommandVersion:0
[junit4] 2> 62936 INFO (qtp1422851308-38) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.S.Request [backuprestore_shard2_replica_t5] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=2
[junit4] 2> 62937 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 62937 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 62937 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 62937 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 62937 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 62938 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 62938 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@70ddacb8 commitCommandVersion:0
[junit4] 2> 62952 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.SolrIndexSearcher Opening [Searcher@582d8e1b[backuprestore_shard1_replica_p3] main]
[junit4] 2> 62968 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.SolrIndexSearcher Opening [Searcher@1dbec7a1[backuprestore_shard2_replica_p6] main]
[junit4] 2> 62969 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 62970 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 63636 INFO (searcherExecutor-18-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard2_replica_p6 s:shard2 c:backuprestore r:core_node12) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrCore [backuprestore_shard2_replica_p6] Registered new searcher Searcher@1dbec7a1[backuprestore_shard2_replica_p6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 63699 INFO (searcherExecutor-19-thread-1-processing-n:127.0.0.1:58560_solr x:backuprestore_shard1_replica_p3 s:shard1 c:backuprestore r:core_node9) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrCore [backuprestore_shard1_replica_p3] Registered new searcher Searcher@582d8e1b[backuprestore_shard1_replica_p3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 64458 INFO (qtp1422851308-32) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.S.Request [backuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 64460 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 64460 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 64460 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 64460 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 64460 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 64480 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@5fd941fa[backuprestore_shard1_replica_t2] main]
[junit4] 2> 64483 INFO (searcherExecutor-22-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard1_replica_t2 s:shard1 c:backuprestore r:core_node8) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore [backuprestore_shard1_replica_t2] Registered new searcher Searcher@5fd941fa[backuprestore_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 65903 INFO (qtp1422851308-32) [n:127.0.0.1:58560_solr c:backuprestore s:shard2 r:core_node11 x:backuprestore_shard2_replica_t5] o.a.s.c.S.Request [backuprestore_shard2_replica_t5] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Slave's generation: 2
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher Slave's version: 1509421264457
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 65906 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@70ddacb8 commitCommandVersion:0
[junit4] 2> 65928 INFO (qtp1422851308-39) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.S.Request [backuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Slave's generation: 2
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher Slave's version: 1509421264415
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 65929 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2c8c1df2 commitCommandVersion:0
[junit4] 2> 65948 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.s.SolrIndexSearcher Opening [Searcher@3aa70f9b[backuprestore_shard2_replica_p6] main]
[junit4] 2> 65948 INFO (indexFetcher-48-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 65951 INFO (searcherExecutor-18-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard2_replica_p6 s:shard2 c:backuprestore r:core_node12) [n:127.0.0.1:41201_solr c:backuprestore s:shard2 r:core_node12 x:backuprestore_shard2_replica_p6] o.a.s.c.SolrCore [backuprestore_shard2_replica_p6] Registered new searcher Searcher@3aa70f9b[backuprestore_shard2_replica_p6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 65953 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.s.SolrIndexSearcher Opening [Searcher@8d8fcd8[backuprestore_shard1_replica_p3] main]
[junit4] 2> 65953 INFO (indexFetcher-47-thread-1) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 65958 INFO (searcherExecutor-19-thread-1-processing-n:127.0.0.1:58560_solr x:backuprestore_shard1_replica_p3 s:shard1 c:backuprestore r:core_node9) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node9 x:backuprestore_shard1_replica_p3] o.a.s.c.SolrCore [backuprestore_shard1_replica_p3] Registered new searcher Searcher@8d8fcd8[backuprestore_shard1_replica_p3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 67456 INFO (qtp1422851308-32) [n:127.0.0.1:58560_solr c:backuprestore s:shard1 r:core_node7 x:backuprestore_shard1_replica_n1] o.a.s.c.S.Request [backuprestore_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 67457 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 67457 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 67457 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 67457 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 67457 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 67471 INFO (indexFetcher-50-thread-1) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4ed43433[backuprestore_shard1_replica_t2] main]
[junit4] 2> 67475 INFO (searcherExecutor-22-thread-1-processing-n:127.0.0.1:41201_solr x:backuprestore_shard1_replica_t2 s:shard1 c:backuprestore r:core_node8) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.c.SolrCore [backuprestore_shard1_replica_t2] Registered new searcher Searcher@4ed43433[backuprestore_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 67500 INFO (qtp1468422210-31) [n:127.0.0.1:41201_solr c:backuprestore s:shard1 r:core_node8 x:backuprestore_shard1_replica_t2] o.a.s.u.p.LogUpdateProcessorFactory [backuprestore_shard1_replica_t2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:58560/solr/backuprestore_shard1_replica_n1/&wt=javabin&version=2}{add=[0 (1582742912532217856), 1 (1582742915467182080), 4 (1582742915491299328), 6 (1582742915497590784), 7 (1582742915514368000), 8 (1582742915646488576), 10 (1582742915649634304), 13 (1582742915684237312), 14 (1582742915689480192), 17 (1582742915
[...truncated too long message...]
ave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptor-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-admin-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-authn-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/byte-buddy-1.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/commons-math3-3.6.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/mockito-core-2.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/objenesis-2.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/analysis/icu/lucene-analyzers-icu-7.2.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/contrib/analysis-extras/lib/icu4j-59.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20171031_033935_3816929525940187101917.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20171031_033935_3814221955659416896517.suites -stdin
[junit4] ERROR: JVM J2 ended with an exception: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20171031_033935_3814842593785503032539.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20171031_033935_3814372273523413635019.syserr
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] at java.lang.Thread.run(Thread.java:748)
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:826: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:770: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build.xml:262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/common-build.xml:556: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1494: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1021: At least one slave process threw an exception, first: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J0-20171031_033935_3321840776022627920860.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J0-20171031_033935_3326211161918442131645.syserr
Total time: 235 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-Tests-7.x #197 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 198 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/198/
No tests ran.
Build Log:
[...truncated 7005 lines...]
ERROR: command execution failed.
ERROR: Step ‘Archive the artifacts’ failed: no workspace for Lucene-Solr-Tests-7.x #198
ERROR: Step ‘Publish JUnit test result report’ failed: no workspace for Lucene-Solr-Tests-7.x #198
ERROR: lucene is offline; cannot locate JDK 1.8 (latest)
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
ERROR: lucene is offline; cannot locate JDK 1.8 (latest)
ERROR: lucene is offline; cannot locate JDK 1.8 (latest)
[JENKINS] Lucene-Solr-Tests-7.x - Build # 197 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/197/
10 tests failed.
FAILED: org.apache.solr.cloud.ShardRoutingTest.test
Error Message:
Error from server at http://127.0.0.1:34143: At least one of the node(s) specified are not currently active, no action taken.
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34143: At least one of the node(s) specified are not currently active, no action taken.
at __randomizedtesting.SeedInfo.seed([858C689B840B3A43:DD857412AF757BB]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:626)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1096)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:875)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:418)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ShardRoutingTest
Error Message:
198 threads leaked from SUITE scope at org.apache.solr.cloud.ShardRoutingTest: 1) Thread[id=4679, name=qtp383150318-4679, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=4599, name=Scheduler-351568579, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=4534, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 4) Thread[id=4607, name=qtp331754181-4607-acceptor-0@17c44e2f-ServerConnector@67347bd3{HTTP/1.1,[http/1.1]}{127.0.0.1:59846}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=4588, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=4730, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 7) Thread[id=4675, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=4697, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-EventThread, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 9) Thread[id=4504, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 10) Thread[id=4493, name=qtp2059076216-4493, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=4584, name=org.eclipse.jetty.server.session.HashSessionManager@255a5ef9Timer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=4764, name=zkCallback-809-thread-4, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=4706, name=updateExecutor-832-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=4523, name=qtp1505717836-4523, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=4748, name=zkCallback-797-thread-4, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=4543, name=searcherExecutor-1935-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=4705, name=searcherExecutor-1990-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=4530, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=4650, name=org.eclipse.jetty.server.session.HashSessionManager@1ff96782Timer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=4754, name=zkCallback-842-thread-2, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=4671, name=searcherExecutor-1979-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=4711, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=4610, name=qtp331754181-4610, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=4756, name=zkCallback-842-thread-4, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=4618, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=4554, name=org.eclipse.jetty.server.session.HashSessionManager@e00fddcTimer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=4647, name=qtp1351042721-4647, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=4769, name=zkCallback-821-thread-5, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=4721, name=org.eclipse.jetty.server.session.HashSessionManager@6510dcd6Timer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=4613, name=org.eclipse.jetty.server.session.HashSessionManager@635d8de3Timer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=4500, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=4681, name=qtp383150318-4681-acceptor-0@603f0db1-ServerConnector@79281ec4{HTTP/1.1,[http/1.1]}{127.0.0.1:50183}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=4762, name=zkCallback-809-thread-2, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=4668, name=watches-829-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=4725, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=4578, name=qtp1094907735-4578-acceptor-0@1d2723af-ServerConnector@32599859{HTTP/1.1,[http/1.1]}{127.0.0.1:37045}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 37) Thread[id=4563, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 38) Thread[id=4595, name=zkCallback-815-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=4682, name=qtp383150318-4682, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 40) Thread[id=4715, name=qtp1837148975-4715, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 41) Thread[id=4698, name=zkCallback-835-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 42) Thread[id=4719, name=qtp1837148975-4719-acceptor-0@38f54143-ServerConnector@2b2cf0f1{HTTP/1.1,[http/1.1]}{127.0.0.1:40817}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 43) Thread[id=4524, name=qtp1505717836-4524, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 44) Thread[id=4511, name=Scheduler-1049167543, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 45) Thread[id=4535, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-EventThread, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 46) Thread[id=4536, name=zkCallback-803-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 47) Thread[id=4773, name=zkCallback-828-thread-5, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 48) Thread[id=4684, name=qtp383150318-4684, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 49) Thread[id=4577, name=qtp1094907735-4577, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 50) Thread[id=4643, name=qtp1351042721-4643, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 51) Thread[id=4488, name=qtp2059076216-4488, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 52) Thread[id=4759, name=zkCallback-815-thread-3, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 53) Thread[id=4519, name=qtp1505717836-4519-acceptor-0@75539a1e-ServerConnector@321d6f5c{HTTP/1.1,[http/1.1]}{127.0.0.1:38072}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 54) Thread[id=4624, name=zkCallback-821-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 55) Thread[id=4776, name=zkCallback-835-thread-4, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 56) Thread[id=4752, name=zkCallback-803-thread-4, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 57) Thread[id=4770, name=zkCallback-828-thread-2, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 58) Thread[id=4579, name=qtp1094907735-4579, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 59) Thread[id=4645, name=qtp1351042721-4645, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 60) Thread[id=4529, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 61) Thread[id=4546, name=qtp21053055-4546, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 62) Thread[id=4696, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 63) Thread[id=4648, name=qtp1351042721-4648, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 64) Thread[id=4766, name=zkCallback-821-thread-2, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 65) Thread[id=4665, name=coreContainerWorkExecutor-1974-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 66) Thread[id=4573, name=searcherExecutor-1946-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 67) Thread[id=4514, name=searcherExecutor-1924-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 68) Thread[id=4718, name=qtp1837148975-4718, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 69) Thread[id=4751, name=zkCallback-803-thread-3, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 70) Thread[id=4608, name=qtp331754181-4608, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 71) Thread[id=4644, name=qtp1351042721-4644-acceptor-0@7ce398e3-ServerConnector@3bd4dc11{HTTP/1.1,[http/1.1]}{127.0.0.1:40270}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 72) Thread[id=4699, name=Thread-1013, state=WAITING, group=TGRP-ShardRoutingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 73) Thread[id=4553, name=qtp21053055-4553, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 74) Thread[id=4717, name=qtp1837148975-4717, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 75) Thread[id=4749, name=zkCallback-797-thread-5, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 76) Thread[id=4753, name=zkCallback-803-thread-5, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 77) Thread[id=4537, name=Thread-968, state=WAITING, group=TGRP-ShardRoutingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 78) Thread[id=4758, name=zkCallback-815-thread-2, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 79) Thread[id=4767, name=zkCallback-821-thread-3, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 80) Thread[id=4558, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 81) Thread[id=4771, name=zkCallback-828-thread-3, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 82) Thread[id=4583, name=qtp1094907735-4583, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 83) Thread[id=4507, name=Thread-959, state=WAITING, group=TGRP-ShardRoutingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 84) Thread[id=4489, name=qtp2059076216-4489-acceptor-0@283458db-ServerConnector@7a134e2f{HTTP/1.1,[http/1.1]}{127.0.0.1:36211}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 85) Thread[id=4633, name=searcherExecutor-1968-thread-1, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 86) Thread[id=4700, name=coreContainerWorkExecutor-1985-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 87) Thread[id=4622, name=TEST-ShardRoutingTest.test-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45137), state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 88) Thread[id=4506, name=zkCallback-797-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 89) Thread[id=4520, name=qtp1505717836-4520, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 90) Thread[id=4525, name=org.eclipse.jetty.server.session.HashSessionManager@38d81390Timer, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 91) Thread[id=4716, name=qtp1837148975-4716, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 92) Thread[id=4550, name=qtp21053055-4550, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 93) Thread[id=4740, name=OverseerCollectionConfigSetProcessor-98916296398602249-127.0.0.1:36211_-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 94) Thread[id=4663, name=zkCallback-828-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 95) Thread[id=4691, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 96) Thread[id=4547, name=qtp21053055-4547, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 97) Thread[id=4636, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 98) Thread[id=4487, name=qtp2059076216-4487, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 99) Thread[id=4491, name=qtp2059076216-4491, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 100) Thread[id=4739, name=OverseerStateUpdate-98916296398602249-127.0.0.1:36211_-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:435) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:202) at java.lang.Thread.run(Thread.java:748) 101) Thread[id=4551, name=qtp21053055-4551, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 102) Thread[id=4726, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 103) Thread[id=4657, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 104) Thread[id=4649, name=qtp1351042721-4649, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 105) Thread[id=4714, name=qtp1837148975-4714, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 106) Thread[id=4692, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 107) Thread[id=4755, name=zkCallback-842-thread-3, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 108) Thread[id=4494, name=qtp2059076216-4494, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 109) Thread[id=4617, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 110) Thread[id=4778, name=ScheduledTrigger-2004-thread-3, state=WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 111) Thread[id=4559, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 112) Thread[id=4549, name=qtp21053055-4549, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 113) Thread[id=4548, name=qtp21053055-4548-acceptor-0@78d4bb24-ServerConnector@3109465d{HTTP/1.1,[http/1.1]}{127.0.0.1:34143}, state=RUNNABLE, group=TGRP-ShardRoutingTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 114) Thread[id=4545, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 115) Thread[id=4512, name=watches-798-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 116) Thread[id=4566, name=Thread-977, state=WAITING, group=TGRP-ShardRoutingTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678) 117) Thread[id=4777, name=zkCallback-835-thread-5, state=TIMED_WAITING, group=TGRP-ShardRoutingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurren
[...truncated too long message...]
rent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 18) Thread[id=6926, name=TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[858C689B840B3A43]-SendThread(127.0.0.1:45890), state=TIMED_WAITING, group=TGRP-ShardSplitTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > 19) Thread[id=6900, name=Thread-1417, state=WAITING, group=TGRP-ShardSplitTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678)
[junit4] > 20) Thread[id=6881, name=qtp1732939943-6881, state=RUNNABLE, group=TGRP-ShardSplitTest]
[junit4] > at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 21) Thread[id=6871, name=Thread-1408, state=WAITING, group=TGRP-ShardSplitTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1678)
[junit4] > 22) Thread[id=6943, name=qtp199264780-6943, state=RUNNABLE, group=TGRP-ShardSplitTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
[junit4] > at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
[junit4] > at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
[junit4] > at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 23) Thread[id=6935, name=searcherExecutor-3171-thread-1, state=WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 24) Thread[id=6874, name=Scheduler-821479584, state=WAITING, group=TGRP-ShardSplitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 25) Thread[id=6884, name=qtp1732939943-6884, state=RUNNABLE, group=TGRP-ShardSplitTest]
[junit4] > at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 26) Thread[id=6915, name=qtp551973238-6915, state=RUNNABLE, group=TGRP-ShardSplitTest]
[junit4] > at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([858C689B840B3A43]:0)
[junit4] Completed [688/745 (5!)] on J0 in 578.91s, 5 tests, 1 failure, 3 errors <<< FAILURES!
[...truncated 47426 lines...]